{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 328, "tracking_summary": 15787}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16212.406, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555abeaa6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abeaa6a40] error while decoding MB 0 34, bytestream 15931 [h264 @ 0x555abea3b6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea3b6c0] error while decoding MB 0 29, bytestream 2914 [h264 @ 0x555abea4f840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea4f840] error while decoding MB 0 31, bytestream 47601 [h264 @ 0x555abf7e5540] left block unavailable for req", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 773, "h264_decode_error_count": 1891, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:46:17+09:00", "last_error_summary": "[h264 @ 0x555abeaa6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abeaa6a40] error while decoding MB 0 34, bytestream 15931 [h264 @ 0x555abea3b6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea3b6c0] error while decoding MB 0 29, bytestream 2914 [h264 @ 0x555abea4f840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea4f840] error while decoding MB 0 31, bytestream 47601 [h264 @ 0x555abf7e5540] left block unavailable for req", "last_exit_at": "2026-07-05T20:37:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 29352.737, "last_started_at": "2026-07-05T20:37:04+09:00", "pid": 144613, "process": {"exists": false, "pid": 144613}, "restart_count": 861, "restart_count_per_hour": 0, "rtsp_timeout_count": 484, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 862}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.869, "sample_count": 2359, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1894500, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:46:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-06T04:46:16+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-06T04:46:17+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-06T04:46:17.005990+09:00", "mtime_epoch": 1783280777.0059903, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:46:17+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9529.145, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.17, "pushed": 3.17, "read": 3.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.476, "count": 94, "max_ms": 25.761, "p95_ms": 22.431}, "gstreamer_push": {"avg_ms": 1.179, "count": 95, "max_ms": 1.641, "p95_ms": 1.323}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.214, "count": 94, "max_ms": 65.58, "p95_ms": 64.356}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.943, "count": 94, "max_ms": 68.638, "p95_ms": 61.754}, "yolo11_640_fire_11class": {"avg_ms": 51.537, "count": 94, "max_ms": 68.034, "p95_ms": 64.732}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.262, "count": 94, "max_ms": 79.269, "p95_ms": 76.32}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.479, "count": 95, "max_ms": 140.101, "p95_ms": 136.401}, "read_interval": {"avg_ms": 316.109, "count": 95, "max_ms": 10124.204, "p95_ms": 152.987}, "triton_wait": {"avg_ms": 57.337, "count": 94, "max_ms": 79.281, "p95_ms": 76.331}, "visualize": {"avg_ms": 4.086, "count": 94, "max_ms": 5.073, "p95_ms": 4.818}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 328, "tracking_summary": 15787}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16212.211, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T04:46:17+09:00", "last_error_summary": "[h264 @ 0x555abeaa6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abeaa6a40] error while decoding MB 0 34, bytestream 15931 [h264 @ 0x555abea3b6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea3b6c0] error while decoding MB 0 29, bytestream 2914 [h264 @ 0x555abea4f840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea4f840] error while decoding MB 0 31, bytestream 47601 [h264 @ 0x555abf7e5540] left block unavailable for req", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 773, "h264_decode_error_count": 1891, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:46:17+09:00", "last_error_summary": "[h264 @ 0x555abeaa6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abeaa6a40] error while decoding MB 0 34, bytestream 15931 [h264 @ 0x555abea3b6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea3b6c0] error while decoding MB 0 29, bytestream 2914 [h264 @ 0x555abea4f840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea4f840] error while decoding MB 0 31, bytestream 47601 [h264 @ 0x555abf7e5540] left block unavailable for req", "last_exit_at": "2026-07-06T04:46:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 29352.762, "last_started_at": "2026-07-05T20:37:04+09:00", "pid": 144613, "process": {"exists": false, "pid": 144613}, "restart_count": 861, "restart_count_per_hour": 0, "rtsp_timeout_count": 484, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 862}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.869, "sample_count": 2360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1894501, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:46:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T04:46:17+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-06T04:46:17+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-06T04:46:17.005990+09:00", "mtime_epoch": 1783280777.0059903, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:46:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9529.145, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.201, "pushed": 3.201, "read": 3.168}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.517, "count": 95, "max_ms": 25.761, "p95_ms": 22.431}, "gstreamer_push": {"avg_ms": 1.182, "count": 96, "max_ms": 1.641, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.159, "count": 95, "max_ms": 65.58, "p95_ms": 64.356}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.949, "count": 95, "max_ms": 68.638, "p95_ms": 61.754}, "yolo11_640_fire_11class": {"avg_ms": 51.471, "count": 95, "max_ms": 68.034, "p95_ms": 64.732}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.211, "count": 95, "max_ms": 79.269, "p95_ms": 76.32}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.313, "count": 96, "max_ms": 140.101, "p95_ms": 136.401}, "read_interval": {"avg_ms": 316.109, "count": 95, "max_ms": 10124.204, "p95_ms": 152.987}, "triton_wait": {"avg_ms": 57.284, "count": 95, "max_ms": 79.281, "p95_ms": 76.331}, "visualize": {"avg_ms": 4.101, "count": 95, "max_ms": 5.434, "p95_ms": 4.819}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 328, "tracking_summary": 15787}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16212.258, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 773, "h264_decode_error_count": 1891, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:46:17+09:00", "last_error_summary": "[h264 @ 0x555abeaa6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abeaa6a40] error while decoding MB 0 34, bytestream 15931 [h264 @ 0x555abea3b6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea3b6c0] error while decoding MB 0 29, bytestream 2914 [h264 @ 0x555abea4f840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea4f840] error while decoding MB 0 31, bytestream 47601 [h264 @ 0x555abf7e5540] left block unavailable for req", "last_exit_at": "2026-07-06T04:46:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 29352.769, "last_started_at": "2026-07-05T20:37:04+09:00", "pid": 144613, "process": {"exists": false, "pid": 144613}, "restart_count": 861, "restart_count_per_hour": 0, "rtsp_timeout_count": 484, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 862}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.869, "sample_count": 2360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1894501, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:46:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T04:46:17+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-06T04:46:17+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-06T04:46:17.005990+09:00", "mtime_epoch": 1783280777.0059903, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:46:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9529.145, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.2, "pushed": 3.2, "read": 3.167}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.517, "count": 95, "max_ms": 25.761, "p95_ms": 22.431}, "gstreamer_push": {"avg_ms": 1.182, "count": 96, "max_ms": 1.641, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.159, "count": 95, "max_ms": 65.58, "p95_ms": 64.356}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.949, "count": 95, "max_ms": 68.638, "p95_ms": 61.754}, "yolo11_640_fire_11class": {"avg_ms": 51.471, "count": 95, "max_ms": 68.034, "p95_ms": 64.732}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.211, "count": 95, "max_ms": 79.269, "p95_ms": 76.32}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.313, "count": 96, "max_ms": 140.101, "p95_ms": 136.401}, "read_interval": {"avg_ms": 316.109, "count": 95, "max_ms": 10124.204, "p95_ms": 152.987}, "triton_wait": {"avg_ms": 57.284, "count": 95, "max_ms": 79.281, "p95_ms": 76.331}, "visualize": {"avg_ms": 4.101, "count": 95, "max_ms": 5.434, "p95_ms": 4.819}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 328, "tracking_summary": 15787}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16320.289, "limit_mb": 8796093022207.996}, "detail": {"pid": 144900, "restart_count": 862, "start_count": 863}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 773, "h264_decode_error_count": 1891, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:46:17+09:00", "last_error_summary": "[h264 @ 0x555abeaa6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abeaa6a40] error while decoding MB 0 34, bytestream 15931 [h264 @ 0x555abea3b6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea3b6c0] error while decoding MB 0 29, bytestream 2914 [h264 @ 0x555abea4f840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555abea4f840] error while decoding MB 0 31, bytestream 47601 [h264 @ 0x555abf7e5540] left block unavailable for req", "last_exit_at": "2026-07-06T04:46:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T04:46:19+09:00", "pid": 144900, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.262, "name": "ffmpeg", "pid": 144900, "rss_mb": 139.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 862, "restart_count_per_hour": 1, "rtsp_timeout_count": 484, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 863}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.869, "sample_count": 2325, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1894501, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:46:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.173, "last_frame_pushed_at": "2026-07-06T04:46:17+09:00", "last_frame_read_age_sec": 4.261, "last_frame_read_at": "2026-07-06T04:46:17+09:00", "mjpeg": {"age_sec": 4.156, "exists": true, "mtime": "2026-07-06T04:46:17.105994+09:00", "mtime_epoch": 1783280777.105994, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:46:21+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9529.145, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.138, "pushed": 2.138, "read": 2.104}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 17.848, "count": 64, "max_ms": 25.761, "p95_ms": 21.148}, "gstreamer_push": {"avg_ms": 1.2, "count": 64, "max_ms": 1.5, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.484, "count": 64, "max_ms": 65.58, "p95_ms": 64.205}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.356, "count": 64, "max_ms": 68.638, "p95_ms": 60.744}, "yolo11_640_fire_11class": {"avg_ms": 52.204, "count": 64, "max_ms": 68.034, "p95_ms": 64.732}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.174, "count": 64, "max_ms": 79.269, "p95_ms": 74.157}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.273, "count": 64, "max_ms": 138.511, "p95_ms": 131.89}, "read_interval": {"avg_ms": 409.363, "count": 63, "max_ms": 10124.204, "p95_ms": 147.795}, "triton_wait": {"avg_ms": 57.277, "count": 64, "max_ms": 79.281, "p95_ms": 74.168}, "visualize": {"avg_ms": 3.834, "count": 64, "max_ms": 5.434, "p95_ms": 4.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 328, "tracking_summary": 15800}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16214.305, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ea7e3e6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e3e6a40] error while decoding MB 0 39, bytestream 4306 [h264 @ 0x55ea7e40edc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e40edc0] error while decoding MB 0 6, bytestream 4411 [h264 @ 0x55ea7e3e6a40] top block unavailable for requested intra mode [h264 @ 0x55ea7e3e6a40] error while decoding MB 10 0, bytestream 5936 [h264 @ 0x55ea7f0908c0] error while decoding MB 8 18, bytestream", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 774, "h264_decode_error_count": 1895, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:49:36+09:00", "last_error_summary": "[h264 @ 0x55ea7e3e6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e3e6a40] error while decoding MB 0 39, bytestream 4306 [h264 @ 0x55ea7e40edc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e40edc0] error while decoding MB 0 6, bytestream 4411 [h264 @ 0x55ea7e3e6a40] top block unavailable for requested intra mode [h264 @ 0x55ea7e3e6a40] error while decoding MB 10 0, bytestream 5936 [h264 @ 0x55ea7f0908c0] error while decoding MB 8 18, bytestream", "last_exit_at": "2026-07-06T04:46:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.288, "last_started_at": "2026-07-06T04:46:19+09:00", "pid": 144900, "process": {"exists": false, "pid": 144900}, "restart_count": 862, "restart_count_per_hour": 1, "rtsp_timeout_count": 486, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 863}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.695, "sample_count": 2171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1896018, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:49:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T04:49:36+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T04:49:36+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-06T04:49:36.488943+09:00", "mtime_epoch": 1783280976.4889426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 109032}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:49:36+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9529.918, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.206, "pushed": 3.206, "read": 3.206}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.226, "count": 96, "max_ms": 26.17, "p95_ms": 21.632}, "gstreamer_push": {"avg_ms": 1.164, "count": 96, "max_ms": 1.84, "p95_ms": 1.297}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.709, "count": 96, "max_ms": 66.638, "p95_ms": 61.066}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.578, "count": 96, "max_ms": 71.877, "p95_ms": 63.501}, "yolo11_640_fire_11class": {"avg_ms": 49.283, "count": 96, "max_ms": 65.582, "p95_ms": 62.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.258, "count": 96, "max_ms": 79.216, "p95_ms": 75.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.083, "count": 96, "max_ms": 137.563, "p95_ms": 131.526}, "read_interval": {"avg_ms": 312.345, "count": 96, "max_ms": 19096.647, "p95_ms": 146.752}, "triton_wait": {"avg_ms": 57.276, "count": 96, "max_ms": 79.228, "p95_ms": 75.314}, "visualize": {"avg_ms": 4.083, "count": 96, "max_ms": 6.091, "p95_ms": 4.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 328, "tracking_summary": 15800}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16213.98, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T04:49:36+09:00", "last_error_summary": "[h264 @ 0x55ea7e3e6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e3e6a40] error while decoding MB 0 39, bytestream 4306 [h264 @ 0x55ea7e40edc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e40edc0] error while decoding MB 0 6, bytestream 4411 [h264 @ 0x55ea7e3e6a40] top block unavailable for requested intra mode [h264 @ 0x55ea7e3e6a40] error while decoding MB 10 0, bytestream 5936 [h264 @ 0x55ea7f0908c0] error while decoding MB 8 18, bytestream", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 774, "h264_decode_error_count": 1895, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:49:36+09:00", "last_error_summary": "[h264 @ 0x55ea7e3e6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e3e6a40] error while decoding MB 0 39, bytestream 4306 [h264 @ 0x55ea7e40edc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e40edc0] error while decoding MB 0 6, bytestream 4411 [h264 @ 0x55ea7e3e6a40] top block unavailable for requested intra mode [h264 @ 0x55ea7e3e6a40] error while decoding MB 10 0, bytestream 5936 [h264 @ 0x55ea7f0908c0] error while decoding MB 8 18, bytestream", "last_exit_at": "2026-07-06T04:49:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.352, "last_started_at": "2026-07-06T04:46:19+09:00", "pid": 144900, "process": {"exists": false, "pid": 144900}, "restart_count": 862, "restart_count_per_hour": 1, "rtsp_timeout_count": 486, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 863}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.695, "sample_count": 2171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1896019, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:49:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T04:49:36+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-06T04:49:36+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-06T04:49:36.488943+09:00", "mtime_epoch": 1783280976.4889426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 109032}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:49:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9529.918, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.201, "pushed": 3.201, "read": 3.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.271, "count": 96, "max_ms": 26.17, "p95_ms": 21.917}, "gstreamer_push": {"avg_ms": 1.164, "count": 96, "max_ms": 1.84, "p95_ms": 1.297}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.73, "count": 96, "max_ms": 66.638, "p95_ms": 61.066}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.565, "count": 96, "max_ms": 71.877, "p95_ms": 63.501}, "yolo11_640_fire_11class": {"avg_ms": 49.304, "count": 96, "max_ms": 65.582, "p95_ms": 62.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.278, "count": 96, "max_ms": 79.216, "p95_ms": 75.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.236, "count": 96, "max_ms": 137.563, "p95_ms": 131.526}, "read_interval": {"avg_ms": 312.345, "count": 96, "max_ms": 19096.647, "p95_ms": 146.752}, "triton_wait": {"avg_ms": 57.296, "count": 96, "max_ms": 79.228, "p95_ms": 75.314}, "visualize": {"avg_ms": 4.101, "count": 96, "max_ms": 6.338, "p95_ms": 4.986}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 328, "tracking_summary": 15800}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16213.973, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 774, "h264_decode_error_count": 1895, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:49:36+09:00", "last_error_summary": "[h264 @ 0x55ea7e3e6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e3e6a40] error while decoding MB 0 39, bytestream 4306 [h264 @ 0x55ea7e40edc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e40edc0] error while decoding MB 0 6, bytestream 4411 [h264 @ 0x55ea7e3e6a40] top block unavailable for requested intra mode [h264 @ 0x55ea7e3e6a40] error while decoding MB 10 0, bytestream 5936 [h264 @ 0x55ea7f0908c0] error while decoding MB 8 18, bytestream", "last_exit_at": "2026-07-06T04:49:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.358, "last_started_at": "2026-07-06T04:46:19+09:00", "pid": 144900, "process": {"exists": false, "pid": 144900}, "restart_count": 862, "restart_count_per_hour": 1, "rtsp_timeout_count": 486, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 863}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.695, "sample_count": 2171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1896019, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:49:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T04:49:36+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T04:49:36+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T04:49:36.488943+09:00", "mtime_epoch": 1783280976.4889426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 109032}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:49:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9529.918, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.213, "pushed": 3.213, "read": 3.18}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.271, "count": 96, "max_ms": 26.17, "p95_ms": 21.917}, "gstreamer_push": {"avg_ms": 1.164, "count": 96, "max_ms": 1.84, "p95_ms": 1.297}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.73, "count": 96, "max_ms": 66.638, "p95_ms": 61.066}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.565, "count": 96, "max_ms": 71.877, "p95_ms": 63.501}, "yolo11_640_fire_11class": {"avg_ms": 49.304, "count": 96, "max_ms": 65.582, "p95_ms": 62.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.278, "count": 96, "max_ms": 79.216, "p95_ms": 75.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.236, "count": 96, "max_ms": 137.563, "p95_ms": 131.526}, "read_interval": {"avg_ms": 314.368, "count": 95, "max_ms": 19096.647, "p95_ms": 146.752}, "triton_wait": {"avg_ms": 57.296, "count": 96, "max_ms": 79.228, "p95_ms": 75.314}, "visualize": {"avg_ms": 4.101, "count": 96, "max_ms": 6.338, "p95_ms": 4.986}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 328, "tracking_summary": 15800}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16322.711, "limit_mb": 8796093022207.996}, "detail": {"pid": 145007, "restart_count": 863, "start_count": 864}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 774, "h264_decode_error_count": 1895, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:49:36+09:00", "last_error_summary": "[h264 @ 0x55ea7e3e6a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e3e6a40] error while decoding MB 0 39, bytestream 4306 [h264 @ 0x55ea7e40edc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ea7e40edc0] error while decoding MB 0 6, bytestream 4411 [h264 @ 0x55ea7e3e6a40] top block unavailable for requested intra mode [h264 @ 0x55ea7e3e6a40] error while decoding MB 10 0, bytestream 5936 [h264 @ 0x55ea7f0908c0] error while decoding MB 8 18, bytestream", "last_exit_at": "2026-07-06T04:49:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T04:49:39+09:00", "pid": 145007, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.922, "name": "ffmpeg", "pid": 145007, "rss_mb": 138.922, "state": "S (sleeping)", "threads": 106}, "restart_count": 863, "restart_count_per_hour": 2, "rtsp_timeout_count": 486, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 864}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.669, "sample_count": 2135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1896019, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:49:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.571, "last_frame_pushed_at": "2026-07-06T04:49:36+09:00", "last_frame_read_age_sec": 4.696, "last_frame_read_at": "2026-07-06T04:49:36+09:00", "mjpeg": {"age_sec": 4.558, "exists": true, "mtime": "2026-07-06T04:49:36.620947+09:00", "mtime_epoch": 1783280976.620947, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 109032}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:49:41+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9529.918, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.968, "pushed": 1.968, "read": 1.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 17.843, "count": 58, "max_ms": 22.733, "p95_ms": 21.316}, "gstreamer_push": {"avg_ms": 1.184, "count": 59, "max_ms": 1.346, "p95_ms": 1.297}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.353, "count": 58, "max_ms": 66.638, "p95_ms": 61.275}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.802, "count": 58, "max_ms": 71.877, "p95_ms": 65.31}, "yolo11_640_fire_11class": {"avg_ms": 50.003, "count": 58, "max_ms": 65.582, "p95_ms": 62.68}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.424, "count": 58, "max_ms": 79.216, "p95_ms": 75.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.898, "count": 59, "max_ms": 132.002, "p95_ms": 130.932}, "read_interval": {"avg_ms": 438.061, "count": 58, "max_ms": 19096.647, "p95_ms": 145.032}, "triton_wait": {"avg_ms": 58.446, "count": 58, "max_ms": 79.228, "p95_ms": 75.314}, "visualize": {"avg_ms": 3.906, "count": 58, "max_ms": 6.338, "p95_ms": 4.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 329, "tracking_summary": 15816}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16221.156, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55aae1c9c680] error while decoding MB 77 9, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 775, "h264_decode_error_count": 1896, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:53:44+09:00", "last_error_summary": "[h264 @ 0x55aae1c9c680] error while decoding MB 77 9, bytestream -9", "last_exit_at": "2026-07-06T04:49:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 245.437, "last_started_at": "2026-07-06T04:49:39+09:00", "pid": 145007, "process": {"exists": false, "pid": 145007}, "restart_count": 863, "restart_count_per_hour": 2, "rtsp_timeout_count": 486, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 864}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.276, "sample_count": 2225, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1897937, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:53:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T04:53:44+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-06T04:53:44+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T04:53:44.572355+09:00", "mtime_epoch": 1783281224.5723553, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 140040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:53:44+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9530.949, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.645, "pushed": 7.645, "read": 7.645}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.336, "count": 229, "max_ms": 26.589, "p95_ms": 21.285}, "gstreamer_push": {"avg_ms": 1.101, "count": 229, "max_ms": 1.27, "p95_ms": 1.189}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.977, "count": 229, "max_ms": 69.212, "p95_ms": 63.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.499, "count": 229, "max_ms": 69.582, "p95_ms": 65.292}, "yolo11_640_fire_11class": {"avg_ms": 49.502, "count": 229, "max_ms": 68.748, "p95_ms": 62.721}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.038, "count": 229, "max_ms": 82.084, "p95_ms": 76.653}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.366, "count": 229, "max_ms": 165.067, "p95_ms": 140.791}, "read_interval": {"avg_ms": 131.068, "count": 229, "max_ms": 176.112, "p95_ms": 153.404}, "triton_wait": {"avg_ms": 57.049, "count": 229, "max_ms": 82.096, "p95_ms": 76.664}, "visualize": {"avg_ms": 4.733, "count": 229, "max_ms": 5.999, "p95_ms": 5.628}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 329, "tracking_summary": 15816}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16220.648, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T04:53:44+09:00", "last_error_summary": "[h264 @ 0x55aae1c9c680] error while decoding MB 77 9, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 775, "h264_decode_error_count": 1896, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:53:44+09:00", "last_error_summary": "[h264 @ 0x55aae1c9c680] error while decoding MB 77 9, bytestream -9", "last_exit_at": "2026-07-06T04:53:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 245.506, "last_started_at": "2026-07-06T04:49:39+09:00", "pid": 145007, "process": {"exists": false, "pid": 145007}, "restart_count": 863, "restart_count_per_hour": 2, "rtsp_timeout_count": 486, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 864}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.276, "sample_count": 2226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1897938, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:53:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T04:53:44+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-06T04:53:44+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T04:53:44.572355+09:00", "mtime_epoch": 1783281224.5723553, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 140040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:53:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9530.949, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.663, "pushed": 7.663, "read": 7.629}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.338, "count": 229, "max_ms": 26.589, "p95_ms": 21.285}, "gstreamer_push": {"avg_ms": 1.101, "count": 229, "max_ms": 1.27, "p95_ms": 1.189}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.982, "count": 229, "max_ms": 69.212, "p95_ms": 63.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.495, "count": 229, "max_ms": 69.582, "p95_ms": 65.292}, "yolo11_640_fire_11class": {"avg_ms": 49.5, "count": 229, "max_ms": 68.748, "p95_ms": 62.721}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.051, "count": 229, "max_ms": 82.084, "p95_ms": 76.653}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.38, "count": 229, "max_ms": 165.067, "p95_ms": 140.791}, "read_interval": {"avg_ms": 131.113, "count": 228, "max_ms": 176.112, "p95_ms": 153.404}, "triton_wait": {"avg_ms": 57.062, "count": 229, "max_ms": 82.096, "p95_ms": 76.664}, "visualize": {"avg_ms": 4.733, "count": 229, "max_ms": 5.999, "p95_ms": 5.628}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 329, "tracking_summary": 15816}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16220.641, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 775, "h264_decode_error_count": 1896, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:53:44+09:00", "last_error_summary": "[h264 @ 0x55aae1c9c680] error while decoding MB 77 9, bytestream -9", "last_exit_at": "2026-07-06T04:53:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 245.511, "last_started_at": "2026-07-06T04:49:39+09:00", "pid": 145007, "process": {"exists": false, "pid": 145007}, "restart_count": 863, "restart_count_per_hour": 2, "rtsp_timeout_count": 486, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 864}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.276, "sample_count": 2226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1897938, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:53:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T04:53:44+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-06T04:53:44+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T04:53:44.572355+09:00", "mtime_epoch": 1783281224.5723553, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 140040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:53:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9530.949, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.661, "pushed": 7.661, "read": 7.628}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.338, "count": 229, "max_ms": 26.589, "p95_ms": 21.285}, "gstreamer_push": {"avg_ms": 1.101, "count": 229, "max_ms": 1.27, "p95_ms": 1.189}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.982, "count": 229, "max_ms": 69.212, "p95_ms": 63.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.495, "count": 229, "max_ms": 69.582, "p95_ms": 65.292}, "yolo11_640_fire_11class": {"avg_ms": 49.5, "count": 229, "max_ms": 68.748, "p95_ms": 62.721}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.051, "count": 229, "max_ms": 82.084, "p95_ms": 76.653}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.38, "count": 229, "max_ms": 165.067, "p95_ms": 140.791}, "read_interval": {"avg_ms": 131.113, "count": 228, "max_ms": 176.112, "p95_ms": 153.404}, "triton_wait": {"avg_ms": 57.062, "count": 229, "max_ms": 82.096, "p95_ms": 76.664}, "visualize": {"avg_ms": 4.733, "count": 229, "max_ms": 5.999, "p95_ms": 5.628}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 329, "tracking_summary": 15816}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16334.359, "limit_mb": 8796093022207.996}, "detail": {"pid": 145174, "restart_count": 864, "start_count": 865}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 775, "h264_decode_error_count": 1896, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:53:44+09:00", "last_error_summary": "[h264 @ 0x55aae1c9c680] error while decoding MB 77 9, bytestream -9", "last_exit_at": "2026-07-06T04:53:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T04:53:47+09:00", "pid": 145174, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.742, "name": "ffmpeg", "pid": 145174, "rss_mb": 144.742, "state": "S (sleeping)", "threads": 106}, "restart_count": 864, "restart_count_per_hour": 3, "rtsp_timeout_count": 486, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 865}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.365, "sample_count": 2177, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1897938, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:53:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.792, "last_frame_pushed_at": "2026-07-06T04:53:44+09:00", "last_frame_read_age_sec": 4.905, "last_frame_read_at": "2026-07-06T04:53:44+09:00", "mjpeg": {"age_sec": 4.778, "exists": true, "mtime": "2026-07-06T04:53:44.692358+09:00", "mtime_epoch": 1783281224.6923585, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 139547}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:53:49+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9530.949, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 193, "pushed_frames": 193, "read_frames": 192, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.436, "pushed": 6.436, "read": 6.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.309, "count": 192, "max_ms": 22.117, "p95_ms": 21.285}, "gstreamer_push": {"avg_ms": 1.101, "count": 193, "max_ms": 1.27, "p95_ms": 1.205}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.031, "count": 192, "max_ms": 69.212, "p95_ms": 63.079}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.543, "count": 192, "max_ms": 68.923, "p95_ms": 65.292}, "yolo11_640_fire_11class": {"avg_ms": 49.476, "count": 192, "max_ms": 68.748, "p95_ms": 62.557}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.205, "count": 192, "max_ms": 82.084, "p95_ms": 76.44}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.502, "count": 193, "max_ms": 165.067, "p95_ms": 141.168}, "read_interval": {"avg_ms": 131.211, "count": 192, "max_ms": 176.112, "p95_ms": 153.695}, "triton_wait": {"avg_ms": 57.216, "count": 192, "max_ms": 82.096, "p95_ms": 76.452}, "visualize": {"avg_ms": 4.733, "count": 192, "max_ms": 5.999, "p95_ms": 5.708}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 329, "tracking_summary": 15835}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16224.363, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556196176dc0] left block unavailable for requested intra mode [h264 @ 0x556196176dc0] error while decoding MB 0 12, bytestream 4565 [h264 @ 0x5561953c4000] cabac decode of qscale diff failed at 28 30 [h264 @ 0x5561953c4000] error while decoding MB 28 30, bytestream 1816 [h264 @ 0x556195f0a240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556195f0a240] error while decoding MB 0 12, bytestream 3517 [h264 @ 0x5561954e0d40] left block unavailable for requested intra4x4 m", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 776, "h264_decode_error_count": 1905, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:58:44+09:00", "last_error_summary": "[h264 @ 0x556196176dc0] left block unavailable for requested intra mode [h264 @ 0x556196176dc0] error while decoding MB 0 12, bytestream 4565 [h264 @ 0x5561953c4000] cabac decode of qscale diff failed at 28 30 [h264 @ 0x5561953c4000] error while decoding MB 28 30, bytestream 1816 [h264 @ 0x556195f0a240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556195f0a240] error while decoding MB 0 12, bytestream 3517 [h264 @ 0x5561954e0d40] left block unavailable for requested intra4x4 m", "last_exit_at": "2026-07-06T04:53:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.894, "last_started_at": "2026-07-06T04:53:47+09:00", "pid": 145174, "process": {"exists": false, "pid": 145174}, "restart_count": 864, "restart_count_per_hour": 3, "rtsp_timeout_count": 488, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 865}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.647, "sample_count": 2296, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1900231, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:58:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T04:58:44+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T04:58:44+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T04:58:44.308181+09:00", "mtime_epoch": 1783281524.3081808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 67078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:58:44+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9531.98, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 109, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.634, "pushed": 3.634, "read": 3.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 16.795, "count": 109, "max_ms": 21.652, "p95_ms": 18.614}, "gstreamer_push": {"avg_ms": 1.243, "count": 109, "max_ms": 1.429, "p95_ms": 1.356}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.671, "count": 109, "max_ms": 65.686, "p95_ms": 63.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.463, "count": 109, "max_ms": 68.471, "p95_ms": 62.975}, "yolo11_640_fire_11class": {"avg_ms": 55.384, "count": 109, "max_ms": 73.751, "p95_ms": 66.902}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.56, "count": 109, "max_ms": 76.202, "p95_ms": 70.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.536, "count": 109, "max_ms": 101.773, "p95_ms": 96.823}, "read_interval": {"avg_ms": 275.373, "count": 109, "max_ms": 19354.578, "p95_ms": 113.357}, "triton_wait": {"avg_ms": 57.414, "count": 109, "max_ms": 76.214, "p95_ms": 72.185}, "visualize": {"avg_ms": 3.485, "count": 109, "max_ms": 4.361, "p95_ms": 3.722}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 329, "tracking_summary": 15835}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16223.699, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T04:58:44+09:00", "last_error_summary": "[h264 @ 0x556196176dc0] left block unavailable for requested intra mode [h264 @ 0x556196176dc0] error while decoding MB 0 12, bytestream 4565 [h264 @ 0x5561953c4000] cabac decode of qscale diff failed at 28 30 [h264 @ 0x5561953c4000] error while decoding MB 28 30, bytestream 1816 [h264 @ 0x556195f0a240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556195f0a240] error while decoding MB 0 12, bytestream 3517 [h264 @ 0x5561954e0d40] left block unavailable for requested intra4x4 m", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 776, "h264_decode_error_count": 1905, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:58:44+09:00", "last_error_summary": "[h264 @ 0x556196176dc0] left block unavailable for requested intra mode [h264 @ 0x556196176dc0] error while decoding MB 0 12, bytestream 4565 [h264 @ 0x5561953c4000] cabac decode of qscale diff failed at 28 30 [h264 @ 0x5561953c4000] error while decoding MB 28 30, bytestream 1816 [h264 @ 0x556195f0a240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556195f0a240] error while decoding MB 0 12, bytestream 3517 [h264 @ 0x5561954e0d40] left block unavailable for requested intra4x4 m", "last_exit_at": "2026-07-06T04:58:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.938, "last_started_at": "2026-07-06T04:53:47+09:00", "pid": 145174, "process": {"exists": false, "pid": 145174}, "restart_count": 864, "restart_count_per_hour": 3, "rtsp_timeout_count": 488, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 865}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.647, "sample_count": 2297, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1900232, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:58:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T04:58:44+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-06T04:58:44+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-06T04:58:44.308181+09:00", "mtime_epoch": 1783281524.3081808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 67078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:58:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9531.98, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.641, "pushed": 3.641, "read": 3.608}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 16.796, "count": 109, "max_ms": 21.652, "p95_ms": 18.614}, "gstreamer_push": {"avg_ms": 1.243, "count": 109, "max_ms": 1.429, "p95_ms": 1.356}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.631, "count": 109, "max_ms": 65.686, "p95_ms": 63.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.42, "count": 109, "max_ms": 68.471, "p95_ms": 62.975}, "yolo11_640_fire_11class": {"avg_ms": 55.468, "count": 109, "max_ms": 73.751, "p95_ms": 66.902}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.598, "count": 109, "max_ms": 76.202, "p95_ms": 70.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.613, "count": 109, "max_ms": 101.773, "p95_ms": 96.823}, "read_interval": {"avg_ms": 277.033, "count": 108, "max_ms": 19354.578, "p95_ms": 113.357}, "triton_wait": {"avg_ms": 57.49, "count": 109, "max_ms": 76.214, "p95_ms": 72.185}, "visualize": {"avg_ms": 3.484, "count": 109, "max_ms": 4.361, "p95_ms": 3.722}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 329, "tracking_summary": 15835}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16223.805, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 776, "h264_decode_error_count": 1905, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:58:44+09:00", "last_error_summary": "[h264 @ 0x556196176dc0] left block unavailable for requested intra mode [h264 @ 0x556196176dc0] error while decoding MB 0 12, bytestream 4565 [h264 @ 0x5561953c4000] cabac decode of qscale diff failed at 28 30 [h264 @ 0x5561953c4000] error while decoding MB 28 30, bytestream 1816 [h264 @ 0x556195f0a240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556195f0a240] error while decoding MB 0 12, bytestream 3517 [h264 @ 0x5561954e0d40] left block unavailable for requested intra4x4 m", "last_exit_at": "2026-07-06T04:58:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.945, "last_started_at": "2026-07-06T04:53:47+09:00", "pid": 145174, "process": {"exists": false, "pid": 145174}, "restart_count": 864, "restart_count_per_hour": 3, "rtsp_timeout_count": 488, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 865}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.647, "sample_count": 2297, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1900232, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:58:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T04:58:44+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-06T04:58:44+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T04:58:44.308181+09:00", "mtime_epoch": 1783281524.3081808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 67078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:58:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9531.98, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.64, "pushed": 3.64, "read": 3.607}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 16.796, "count": 109, "max_ms": 21.652, "p95_ms": 18.614}, "gstreamer_push": {"avg_ms": 1.243, "count": 109, "max_ms": 1.429, "p95_ms": 1.356}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.631, "count": 109, "max_ms": 65.686, "p95_ms": 63.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.42, "count": 109, "max_ms": 68.471, "p95_ms": 62.975}, "yolo11_640_fire_11class": {"avg_ms": 55.468, "count": 109, "max_ms": 73.751, "p95_ms": 66.902}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.598, "count": 109, "max_ms": 76.202, "p95_ms": 70.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.613, "count": 109, "max_ms": 101.773, "p95_ms": 96.823}, "read_interval": {"avg_ms": 277.033, "count": 108, "max_ms": 19354.578, "p95_ms": 113.357}, "triton_wait": {"avg_ms": 57.49, "count": 109, "max_ms": 76.214, "p95_ms": 72.185}, "visualize": {"avg_ms": 3.484, "count": 109, "max_ms": 4.361, "p95_ms": 3.722}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 329, "tracking_summary": 15835}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16331.758, "limit_mb": 8796093022207.996}, "detail": {"pid": 145281, "restart_count": 865, "start_count": 866}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 776, "h264_decode_error_count": 1905, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T04:58:44+09:00", "last_error_summary": "[h264 @ 0x556196176dc0] left block unavailable for requested intra mode [h264 @ 0x556196176dc0] error while decoding MB 0 12, bytestream 4565 [h264 @ 0x5561953c4000] cabac decode of qscale diff failed at 28 30 [h264 @ 0x5561953c4000] error while decoding MB 28 30, bytestream 1816 [h264 @ 0x556195f0a240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556195f0a240] error while decoding MB 0 12, bytestream 3517 [h264 @ 0x5561954e0d40] left block unavailable for requested intra4x4 m", "last_exit_at": "2026-07-06T04:58:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T04:58:47+09:00", "pid": 145281, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.074, "name": "ffmpeg", "pid": 145281, "rss_mb": 139.074, "state": "S (sleeping)", "threads": 106}, "restart_count": 865, "restart_count_per_hour": 4, "rtsp_timeout_count": 488, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 866}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.647, "sample_count": 2294, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1900232, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T04:58:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.938, "last_frame_pushed_at": "2026-07-06T04:58:44+09:00", "last_frame_read_age_sec": 5.029, "last_frame_read_at": "2026-07-06T04:58:44+09:00", "mjpeg": {"age_sec": 4.925, "exists": true, "mtime": "2026-07-06T04:58:44.412184+09:00", "mtime_epoch": 1783281524.4121835, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 74372}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T04:58:49+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9531.98, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.003, "pushed": 2.003, "read": 1.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 16.767, "count": 60, "max_ms": 19.399, "p95_ms": 17.957}, "gstreamer_push": {"avg_ms": 1.25, "count": 60, "max_ms": 1.429, "p95_ms": 1.349}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.348, "count": 60, "max_ms": 65.686, "p95_ms": 63.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.022, "count": 60, "max_ms": 68.471, "p95_ms": 62.975}, "yolo11_640_fire_11class": {"avg_ms": 56.003, "count": 60, "max_ms": 70.7, "p95_ms": 68.685}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.486, "count": 60, "max_ms": 76.202, "p95_ms": 69.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.942, "count": 60, "max_ms": 101.773, "p95_ms": 95.667}, "read_interval": {"avg_ms": 424.076, "count": 59, "max_ms": 19354.578, "p95_ms": 113.841}, "triton_wait": {"avg_ms": 57.826, "count": 60, "max_ms": 76.214, "p95_ms": 70.71}, "visualize": {"avg_ms": 3.481, "count": 60, "max_ms": 4.361, "p95_ms": 3.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15876}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16233.004, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55de4c858ec0] left block unavailable for requested intra mode [h264 @ 0x55de4c858ec0] error while decoding MB 0 12, bytestream 1722 [h264 @ 0x55de4c8adb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55de4c8adb00] error while decoding MB 0 12, bytestream 4122 [h264 @ 0x55de4c974a80] left block unavailable for requested intra mode [h264 @ 0x55de4c974a80] error while decoding MB 0 21, bytestream 509 [in#0/rtsp @ 0x55de4c828e00] Error during demuxing: Connection timed ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 777, "h264_decode_error_count": 1908, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:07:57+09:00", "last_error_summary": "[h264 @ 0x55de4c858ec0] left block unavailable for requested intra mode [h264 @ 0x55de4c858ec0] error while decoding MB 0 12, bytestream 1722 [h264 @ 0x55de4c8adb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55de4c8adb00] error while decoding MB 0 12, bytestream 4122 [h264 @ 0x55de4c974a80] left block unavailable for requested intra mode [h264 @ 0x55de4c974a80] error while decoding MB 0 21, bytestream 509 [in#0/rtsp @ 0x55de4c828e00] Error during demuxing: Connection timed ", "last_exit_at": "2026-07-06T04:58:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 549.733, "last_started_at": "2026-07-06T04:58:47+09:00", "pid": 145281, "process": {"exists": false, "pid": 145281}, "restart_count": 865, "restart_count_per_hour": 4, "rtsp_timeout_count": 490, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 866}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.671, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1905131, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:07:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.054, "last_frame_pushed_at": "2026-07-06T05:07:57+09:00", "last_frame_read_age_sec": 0.044, "last_frame_read_at": "2026-07-06T05:07:57+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T05:07:57.024975+09:00", "mtime_epoch": 1783282077.0249746, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:07:57+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9534.301, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 85, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.843, "pushed": 2.843, "read": 2.843}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.215, "count": 85, "max_ms": 28.482, "p95_ms": 21.316}, "gstreamer_push": {"avg_ms": 1.103, "count": 85, "max_ms": 1.209, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.361, "count": 85, "max_ms": 62.048, "p95_ms": 60.369}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.662, "count": 85, "max_ms": 77.417, "p95_ms": 62.409}, "yolo11_640_fire_11class": {"avg_ms": 47.649, "count": 85, "max_ms": 62.897, "p95_ms": 60.543}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.092, "count": 85, "max_ms": 78.466, "p95_ms": 75.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.773, "count": 85, "max_ms": 140.542, "p95_ms": 138.519}, "read_interval": {"avg_ms": 352.449, "count": 85, "max_ms": 10124.088, "p95_ms": 153.805}, "triton_wait": {"avg_ms": 56.104, "count": 85, "max_ms": 78.478, "p95_ms": 75.675}, "visualize": {"avg_ms": 4.73, "count": 85, "max_ms": 5.269, "p95_ms": 4.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15876}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16232.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:07:57+09:00", "last_error_summary": "[h264 @ 0x55de4c858ec0] left block unavailable for requested intra mode [h264 @ 0x55de4c858ec0] error while decoding MB 0 12, bytestream 1722 [h264 @ 0x55de4c8adb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55de4c8adb00] error while decoding MB 0 12, bytestream 4122 [h264 @ 0x55de4c974a80] left block unavailable for requested intra mode [h264 @ 0x55de4c974a80] error while decoding MB 0 21, bytestream 509 [in#0/rtsp @ 0x55de4c828e00] Error during demuxing: Connection timed ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 777, "h264_decode_error_count": 1908, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:07:57+09:00", "last_error_summary": "[h264 @ 0x55de4c858ec0] left block unavailable for requested intra mode [h264 @ 0x55de4c858ec0] error while decoding MB 0 12, bytestream 1722 [h264 @ 0x55de4c8adb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55de4c8adb00] error while decoding MB 0 12, bytestream 4122 [h264 @ 0x55de4c974a80] left block unavailable for requested intra mode [h264 @ 0x55de4c974a80] error while decoding MB 0 21, bytestream 509 [in#0/rtsp @ 0x55de4c828e00] Error during demuxing: Connection timed ", "last_exit_at": "2026-07-06T05:07:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 549.816, "last_started_at": "2026-07-06T04:58:47+09:00", "pid": 145281, "process": {"exists": false, "pid": 145281}, "restart_count": 865, "restart_count_per_hour": 4, "rtsp_timeout_count": 490, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 866}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.671, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1905132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:07:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T05:07:57+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T05:07:57+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-06T05:07:57.024975+09:00", "mtime_epoch": 1783282077.0249746, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:07:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9534.301, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.869, "pushed": 2.869, "read": 2.836}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.205, "count": 85, "max_ms": 28.482, "p95_ms": 21.316}, "gstreamer_push": {"avg_ms": 1.104, "count": 86, "max_ms": 1.209, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.469, "count": 85, "max_ms": 62.048, "p95_ms": 60.369}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.756, "count": 85, "max_ms": 77.417, "p95_ms": 62.409}, "yolo11_640_fire_11class": {"avg_ms": 47.718, "count": 85, "max_ms": 62.897, "p95_ms": 60.543}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.216, "count": 85, "max_ms": 78.466, "p95_ms": 75.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.813, "count": 86, "max_ms": 140.542, "p95_ms": 138.519}, "read_interval": {"avg_ms": 352.449, "count": 85, "max_ms": 10124.088, "p95_ms": 153.805}, "triton_wait": {"avg_ms": 56.228, "count": 85, "max_ms": 78.478, "p95_ms": 75.675}, "visualize": {"avg_ms": 4.734, "count": 85, "max_ms": 5.269, "p95_ms": 4.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15876}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16232.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 777, "h264_decode_error_count": 1908, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:07:57+09:00", "last_error_summary": "[h264 @ 0x55de4c858ec0] left block unavailable for requested intra mode [h264 @ 0x55de4c858ec0] error while decoding MB 0 12, bytestream 1722 [h264 @ 0x55de4c8adb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55de4c8adb00] error while decoding MB 0 12, bytestream 4122 [h264 @ 0x55de4c974a80] left block unavailable for requested intra mode [h264 @ 0x55de4c974a80] error while decoding MB 0 21, bytestream 509 [in#0/rtsp @ 0x55de4c828e00] Error during demuxing: Connection timed ", "last_exit_at": "2026-07-06T05:07:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 549.821, "last_started_at": "2026-07-06T04:58:47+09:00", "pid": 145281, "process": {"exists": false, "pid": 145281}, "restart_count": 865, "restart_count_per_hour": 4, "rtsp_timeout_count": 490, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 866}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.671, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1905132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:07:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T05:07:57+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T05:07:57+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-06T05:07:57.024975+09:00", "mtime_epoch": 1783282077.0249746, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:07:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9534.301, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.868, "pushed": 2.868, "read": 2.835}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.205, "count": 85, "max_ms": 28.482, "p95_ms": 21.316}, "gstreamer_push": {"avg_ms": 1.104, "count": 86, "max_ms": 1.209, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.469, "count": 85, "max_ms": 62.048, "p95_ms": 60.369}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.756, "count": 85, "max_ms": 77.417, "p95_ms": 62.409}, "yolo11_640_fire_11class": {"avg_ms": 47.718, "count": 85, "max_ms": 62.897, "p95_ms": 60.543}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.216, "count": 85, "max_ms": 78.466, "p95_ms": 75.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.813, "count": 86, "max_ms": 140.542, "p95_ms": 138.519}, "read_interval": {"avg_ms": 352.449, "count": 85, "max_ms": 10124.088, "p95_ms": 153.805}, "triton_wait": {"avg_ms": 56.228, "count": 85, "max_ms": 78.478, "p95_ms": 75.675}, "visualize": {"avg_ms": 4.734, "count": 85, "max_ms": 5.269, "p95_ms": 4.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15876}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16340.281, "limit_mb": 8796093022207.996}, "detail": {"pid": 145448, "restart_count": 866, "start_count": 867}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 777, "h264_decode_error_count": 1908, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:07:57+09:00", "last_error_summary": "[h264 @ 0x55de4c858ec0] left block unavailable for requested intra mode [h264 @ 0x55de4c858ec0] error while decoding MB 0 12, bytestream 1722 [h264 @ 0x55de4c8adb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55de4c8adb00] error while decoding MB 0 12, bytestream 4122 [h264 @ 0x55de4c974a80] left block unavailable for requested intra mode [h264 @ 0x55de4c974a80] error while decoding MB 0 21, bytestream 509 [in#0/rtsp @ 0x55de4c828e00] Error during demuxing: Connection timed ", "last_exit_at": "2026-07-06T05:07:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:07:59+09:00", "pid": 145448, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.312, "name": "ffmpeg", "pid": 145448, "rss_mb": 139.312, "state": "S (sleeping)", "threads": 106}, "restart_count": 866, "restart_count_per_hour": 5, "rtsp_timeout_count": 490, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 867}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.709, "sample_count": 2452, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1905132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:07:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.697, "last_frame_pushed_at": "2026-07-06T05:07:57+09:00", "last_frame_read_age_sec": 4.82, "last_frame_read_at": "2026-07-06T05:07:57+09:00", "mjpeg": {"age_sec": 4.679, "exists": true, "mtime": "2026-07-06T05:07:57.164978+09:00", "mtime_epoch": 1783282077.1649776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 128993}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:08:01+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9534.301, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.669, "pushed": 1.669, "read": 1.636}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.202, "count": 50, "max_ms": 23.552, "p95_ms": 21.237}, "gstreamer_push": {"avg_ms": 1.111, "count": 50, "max_ms": 1.209, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.003, "count": 50, "max_ms": 62.048, "p95_ms": 60.369}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.408, "count": 50, "max_ms": 67.542, "p95_ms": 62.409}, "yolo11_640_fire_11class": {"avg_ms": 47.535, "count": 50, "max_ms": 62.897, "p95_ms": 61.019}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.187, "count": 50, "max_ms": 78.466, "p95_ms": 76.214}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.779, "count": 50, "max_ms": 139.35, "p95_ms": 138.765}, "read_interval": {"avg_ms": 515.248, "count": 49, "max_ms": 10124.088, "p95_ms": 154.724}, "triton_wait": {"avg_ms": 56.199, "count": 50, "max_ms": 78.478, "p95_ms": 76.226}, "visualize": {"avg_ms": 4.725, "count": 50, "max_ms": 5.18, "p95_ms": 4.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15888}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16234.617, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a860c11a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860c11a00] error while decoding MB 0 7, bytestream 4883 [h264 @ 0x55a86170f680] error while decoding MB 8 21, bytestream -7 [h264 @ 0x55a860b245c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860b245c0] error while decoding MB 0 17, bytestream 6715 [h264 @ 0x55a860c25bc0] error while decoding MB 64 19, bytestream -8 [in#0/rtsp @ 0x55a860af4e00] Error during demuxing: Connection timed", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 778, "h264_decode_error_count": 1912, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:11:16+09:00", "last_error_summary": "[h264 @ 0x55a860c11a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860c11a00] error while decoding MB 0 7, bytestream 4883 [h264 @ 0x55a86170f680] error while decoding MB 8 21, bytestream -7 [h264 @ 0x55a860b245c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860b245c0] error while decoding MB 0 17, bytestream 6715 [h264 @ 0x55a860c25bc0] error while decoding MB 64 19, bytestream -8 [in#0/rtsp @ 0x55a860af4e00] Error during demuxing: Connection timed", "last_exit_at": "2026-07-06T05:07:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.053, "last_started_at": "2026-07-06T05:07:59+09:00", "pid": 145448, "process": {"exists": false, "pid": 145448}, "restart_count": 866, "restart_count_per_hour": 5, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 867}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.797, "sample_count": 2170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1906575, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:11:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-06T05:11:16+09:00", "last_frame_read_age_sec": 0.044, "last_frame_read_at": "2026-07-06T05:11:16+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T05:11:16.857339+09:00", "mtime_epoch": 1783282276.8573387, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 133376}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:11:16+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9535.074, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 85, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.84, "pushed": 2.84, "read": 2.84}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.568, "count": 85, "max_ms": 20.277, "p95_ms": 20.006}, "gstreamer_push": {"avg_ms": 1.136, "count": 85, "max_ms": 1.584, "p95_ms": 1.265}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.322, "count": 85, "max_ms": 67.113, "p95_ms": 63.075}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.672, "count": 85, "max_ms": 75.706, "p95_ms": 70.658}, "yolo11_640_fire_11class": {"avg_ms": 50.423, "count": 85, "max_ms": 66.591, "p95_ms": 63.215}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.273, "count": 85, "max_ms": 81.255, "p95_ms": 76.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.947, "count": 85, "max_ms": 140.363, "p95_ms": 137.374}, "read_interval": {"avg_ms": 353.025, "count": 85, "max_ms": 10126.019, "p95_ms": 151.823}, "triton_wait": {"avg_ms": 56.418, "count": 85, "max_ms": 81.267, "p95_ms": 76.083}, "visualize": {"avg_ms": 4.567, "count": 85, "max_ms": 5.638, "p95_ms": 5.09}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15888}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16234.105, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:11:16+09:00", "last_error_summary": "[h264 @ 0x55a860c11a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860c11a00] error while decoding MB 0 7, bytestream 4883 [h264 @ 0x55a86170f680] error while decoding MB 8 21, bytestream -7 [h264 @ 0x55a860b245c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860b245c0] error while decoding MB 0 17, bytestream 6715 [h264 @ 0x55a860c25bc0] error while decoding MB 64 19, bytestream -8 [in#0/rtsp @ 0x55a860af4e00] Error during demuxing: Connection timed", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 778, "h264_decode_error_count": 1912, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:11:16+09:00", "last_error_summary": "[h264 @ 0x55a860c11a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860c11a00] error while decoding MB 0 7, bytestream 4883 [h264 @ 0x55a86170f680] error while decoding MB 8 21, bytestream -7 [h264 @ 0x55a860b245c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860b245c0] error while decoding MB 0 17, bytestream 6715 [h264 @ 0x55a860c25bc0] error while decoding MB 64 19, bytestream -8 [in#0/rtsp @ 0x55a860af4e00] Error during demuxing: Connection timed", "last_exit_at": "2026-07-06T05:11:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.133, "last_started_at": "2026-07-06T05:07:59+09:00", "pid": 145448, "process": {"exists": false, "pid": 145448}, "restart_count": 866, "restart_count_per_hour": 5, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 867}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.797, "sample_count": 2170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1906576, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:11:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T05:11:16+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-06T05:11:16+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T05:11:16.857339+09:00", "mtime_epoch": 1783282276.8573387, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 133376}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:11:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9535.074, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.844, "pushed": 2.844, "read": 2.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.605, "count": 85, "max_ms": 22.821, "p95_ms": 20.02}, "gstreamer_push": {"avg_ms": 1.136, "count": 85, "max_ms": 1.584, "p95_ms": 1.265}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.322, "count": 85, "max_ms": 67.113, "p95_ms": 63.075}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.631, "count": 85, "max_ms": 75.706, "p95_ms": 70.658}, "yolo11_640_fire_11class": {"avg_ms": 50.431, "count": 85, "max_ms": 66.591, "p95_ms": 63.215}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.287, "count": 85, "max_ms": 81.255, "p95_ms": 76.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.942, "count": 85, "max_ms": 140.363, "p95_ms": 137.374}, "read_interval": {"avg_ms": 355.677, "count": 84, "max_ms": 10126.019, "p95_ms": 151.823}, "triton_wait": {"avg_ms": 56.398, "count": 85, "max_ms": 81.267, "p95_ms": 76.083}, "visualize": {"avg_ms": 4.562, "count": 85, "max_ms": 5.638, "p95_ms": 4.913}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15888}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16234.18, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 778, "h264_decode_error_count": 1912, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:11:16+09:00", "last_error_summary": "[h264 @ 0x55a860c11a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860c11a00] error while decoding MB 0 7, bytestream 4883 [h264 @ 0x55a86170f680] error while decoding MB 8 21, bytestream -7 [h264 @ 0x55a860b245c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860b245c0] error while decoding MB 0 17, bytestream 6715 [h264 @ 0x55a860c25bc0] error while decoding MB 64 19, bytestream -8 [in#0/rtsp @ 0x55a860af4e00] Error during demuxing: Connection timed", "last_exit_at": "2026-07-06T05:11:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.138, "last_started_at": "2026-07-06T05:07:59+09:00", "pid": 145448, "process": {"exists": false, "pid": 145448}, "restart_count": 866, "restart_count_per_hour": 5, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 867}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.797, "sample_count": 2170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1906576, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:11:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T05:11:16+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T05:11:16+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T05:11:16.857339+09:00", "mtime_epoch": 1783282276.8573387, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 133376}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:11:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9535.074, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.843, "pushed": 2.843, "read": 2.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.605, "count": 85, "max_ms": 22.821, "p95_ms": 20.02}, "gstreamer_push": {"avg_ms": 1.136, "count": 85, "max_ms": 1.584, "p95_ms": 1.265}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.322, "count": 85, "max_ms": 67.113, "p95_ms": 63.075}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.631, "count": 85, "max_ms": 75.706, "p95_ms": 70.658}, "yolo11_640_fire_11class": {"avg_ms": 50.431, "count": 85, "max_ms": 66.591, "p95_ms": 63.215}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.287, "count": 85, "max_ms": 81.255, "p95_ms": 76.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.942, "count": 85, "max_ms": 140.363, "p95_ms": 137.374}, "read_interval": {"avg_ms": 355.677, "count": 84, "max_ms": 10126.019, "p95_ms": 151.823}, "triton_wait": {"avg_ms": 56.398, "count": 85, "max_ms": 81.267, "p95_ms": 76.083}, "visualize": {"avg_ms": 4.562, "count": 85, "max_ms": 5.638, "p95_ms": 4.913}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15888}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16341.418, "limit_mb": 8796093022207.996}, "detail": {"pid": 145555, "restart_count": 867, "start_count": 868}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 778, "h264_decode_error_count": 1912, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:11:16+09:00", "last_error_summary": "[h264 @ 0x55a860c11a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860c11a00] error while decoding MB 0 7, bytestream 4883 [h264 @ 0x55a86170f680] error while decoding MB 8 21, bytestream -7 [h264 @ 0x55a860b245c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a860b245c0] error while decoding MB 0 17, bytestream 6715 [h264 @ 0x55a860c25bc0] error while decoding MB 64 19, bytestream -8 [in#0/rtsp @ 0x55a860af4e00] Error during demuxing: Connection timed", "last_exit_at": "2026-07-06T05:11:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:11:19+09:00", "pid": 145555, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.719, "name": "ffmpeg", "pid": 145555, "rss_mb": 138.719, "state": "S (sleeping)", "threads": 106}, "restart_count": 867, "restart_count_per_hour": 6, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 868}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.752, "sample_count": 2127, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1906576, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:11:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.899, "last_frame_pushed_at": "2026-07-06T05:11:16+09:00", "last_frame_read_age_sec": 5.018, "last_frame_read_at": "2026-07-06T05:11:16+09:00", "mjpeg": {"age_sec": 4.886, "exists": true, "mtime": "2026-07-06T05:11:16.985342+09:00", "mtime_epoch": 1783282276.9853415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131368}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:11:21+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9535.074, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 47, "pushed_frames": 47, "read_frames": 46, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.571, "pushed": 1.571, "read": 1.537}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.477, "count": 47, "max_ms": 22.821, "p95_ms": 20.205}, "gstreamer_push": {"avg_ms": 1.145, "count": 47, "max_ms": 1.584, "p95_ms": 1.292}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.774, "count": 47, "max_ms": 66.578, "p95_ms": 63.075}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.824, "count": 47, "max_ms": 74.192, "p95_ms": 70.658}, "yolo11_640_fire_11class": {"avg_ms": 50.467, "count": 47, "max_ms": 66.591, "p95_ms": 61.978}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.383, "count": 47, "max_ms": 76.865, "p95_ms": 75.263}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.128, "count": 47, "max_ms": 138.588, "p95_ms": 136.299}, "read_interval": {"avg_ms": 543.817, "count": 46, "max_ms": 10126.019, "p95_ms": 151.823}, "triton_wait": {"avg_ms": 57.574, "count": 47, "max_ms": 76.877, "p95_ms": 75.275}, "visualize": {"avg_ms": 4.489, "count": 47, "max_ms": 5.638, "p95_ms": 5.161}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15900}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16237.277, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560f60fd0340] left block unavailable for requested intra mode [h264 @ 0x560f60fd0340] error while decoding MB 0 13, bytestream 5139", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 779, "h264_decode_error_count": 1913, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:14:47+09:00", "last_error_summary": "[h264 @ 0x560f60fd0340] left block unavailable for requested intra mode [h264 @ 0x560f60fd0340] error while decoding MB 0 13, bytestream 5139", "last_exit_at": "2026-07-06T05:11:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 207.697, "last_started_at": "2026-07-06T05:11:19+09:00", "pid": 145555, "process": {"exists": false, "pid": 145555}, "restart_count": 867, "restart_count_per_hour": 6, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 868}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.556, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1908118, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:14:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T05:14:47+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T05:14:47+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-06T05:14:47.529862+09:00", "mtime_epoch": 1783282487.5298624, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149777}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:14:47+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9535.848, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.662, "pushed": 7.662, "read": 7.662}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.092, "count": 229, "max_ms": 25.614, "p95_ms": 21.253}, "gstreamer_push": {"avg_ms": 1.124, "count": 229, "max_ms": 1.701, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.743, "count": 229, "max_ms": 65.008, "p95_ms": 60.197}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.29, "count": 229, "max_ms": 78.235, "p95_ms": 70.456}, "yolo11_640_fire_11class": {"avg_ms": 51.739, "count": 229, "max_ms": 71.264, "p95_ms": 64.525}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.849, "count": 229, "max_ms": 79.424, "p95_ms": 74.043}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.065, "count": 229, "max_ms": 262.132, "p95_ms": 136.684}, "read_interval": {"avg_ms": 130.783, "count": 229, "max_ms": 274.027, "p95_ms": 149.066}, "triton_wait": {"avg_ms": 56.918, "count": 229, "max_ms": 79.436, "p95_ms": 74.054}, "visualize": {"avg_ms": 4.714, "count": 229, "max_ms": 7.044, "p95_ms": 5.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15900}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16236.73, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:14:47+09:00", "last_error_summary": "[h264 @ 0x560f60fd0340] left block unavailable for requested intra mode [h264 @ 0x560f60fd0340] error while decoding MB 0 13, bytestream 5139", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 779, "h264_decode_error_count": 1913, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:14:47+09:00", "last_error_summary": "[h264 @ 0x560f60fd0340] left block unavailable for requested intra mode [h264 @ 0x560f60fd0340] error while decoding MB 0 13, bytestream 5139", "last_exit_at": "2026-07-06T05:14:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 207.784, "last_started_at": "2026-07-06T05:11:19+09:00", "pid": 145555, "process": {"exists": false, "pid": 145555}, "restart_count": 867, "restart_count_per_hour": 6, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 868}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.556, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1908119, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:14:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T05:14:47+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T05:14:47+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-06T05:14:47.529862+09:00", "mtime_epoch": 1783282487.5298624, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149777}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:14:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9535.848, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 230, "pushed_frames": 230, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.673, "pushed": 7.673, "read": 7.639}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.092, "count": 229, "max_ms": 25.614, "p95_ms": 21.253}, "gstreamer_push": {"avg_ms": 1.124, "count": 230, "max_ms": 1.701, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.787, "count": 229, "max_ms": 65.008, "p95_ms": 60.197}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.383, "count": 229, "max_ms": 78.235, "p95_ms": 70.522}, "yolo11_640_fire_11class": {"avg_ms": 51.773, "count": 229, "max_ms": 71.264, "p95_ms": 64.525}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.933, "count": 229, "max_ms": 79.424, "p95_ms": 74.043}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.124, "count": 230, "max_ms": 262.132, "p95_ms": 136.684}, "read_interval": {"avg_ms": 130.783, "count": 229, "max_ms": 274.027, "p95_ms": 149.066}, "triton_wait": {"avg_ms": 57.006, "count": 229, "max_ms": 79.436, "p95_ms": 74.054}, "visualize": {"avg_ms": 4.715, "count": 229, "max_ms": 7.044, "p95_ms": 5.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15900}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16236.82, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 779, "h264_decode_error_count": 1913, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:14:47+09:00", "last_error_summary": "[h264 @ 0x560f60fd0340] left block unavailable for requested intra mode [h264 @ 0x560f60fd0340] error while decoding MB 0 13, bytestream 5139", "last_exit_at": "2026-07-06T05:14:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 207.79, "last_started_at": "2026-07-06T05:11:19+09:00", "pid": 145555, "process": {"exists": false, "pid": 145555}, "restart_count": 867, "restart_count_per_hour": 6, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 868}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.556, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1908119, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:14:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T05:14:47+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T05:14:47+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T05:14:47.529862+09:00", "mtime_epoch": 1783282487.5298624, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149777}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:14:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9535.848, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 230, "pushed_frames": 230, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.671, "pushed": 7.671, "read": 7.638}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.092, "count": 229, "max_ms": 25.614, "p95_ms": 21.253}, "gstreamer_push": {"avg_ms": 1.124, "count": 230, "max_ms": 1.701, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.787, "count": 229, "max_ms": 65.008, "p95_ms": 60.197}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.383, "count": 229, "max_ms": 78.235, "p95_ms": 70.522}, "yolo11_640_fire_11class": {"avg_ms": 51.773, "count": 229, "max_ms": 71.264, "p95_ms": 64.525}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.933, "count": 229, "max_ms": 79.424, "p95_ms": 74.043}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.124, "count": 230, "max_ms": 262.132, "p95_ms": 136.684}, "read_interval": {"avg_ms": 130.783, "count": 229, "max_ms": 274.027, "p95_ms": 149.066}, "triton_wait": {"avg_ms": 57.006, "count": 229, "max_ms": 79.436, "p95_ms": 74.054}, "visualize": {"avg_ms": 4.715, "count": 229, "max_ms": 7.044, "p95_ms": 5.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15900}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16344.711, "limit_mb": 8796093022207.996}, "detail": {"pid": 145662, "restart_count": 868, "start_count": 869}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 779, "h264_decode_error_count": 1913, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:14:47+09:00", "last_error_summary": "[h264 @ 0x560f60fd0340] left block unavailable for requested intra mode [h264 @ 0x560f60fd0340] error while decoding MB 0 13, bytestream 5139", "last_exit_at": "2026-07-06T05:14:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:14:50+09:00", "pid": 145662, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.016, "name": "ffmpeg", "pid": 145662, "rss_mb": 139.016, "state": "S (sleeping)", "threads": 106}, "restart_count": 868, "restart_count_per_hour": 7, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 869}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.77, "sample_count": 2076, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1908119, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:14:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.375, "last_frame_pushed_at": "2026-07-06T05:14:47+09:00", "last_frame_read_age_sec": 4.509, "last_frame_read_at": "2026-07-06T05:14:47+09:00", "mjpeg": {"age_sec": 4.359, "exists": true, "mtime": "2026-07-06T05:14:47.665865+09:00", "mtime_epoch": 1783282487.6658652, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149867}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:14:52+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9535.848, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.552, "pushed": 6.552, "read": 6.519}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.053, "count": 196, "max_ms": 25.614, "p95_ms": 20.915}, "gstreamer_push": {"avg_ms": 1.125, "count": 196, "max_ms": 1.701, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.659, "count": 196, "max_ms": 65.008, "p95_ms": 60.367}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.541, "count": 196, "max_ms": 78.235, "p95_ms": 70.973}, "yolo11_640_fire_11class": {"avg_ms": 52.056, "count": 196, "max_ms": 71.264, "p95_ms": 64.705}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.051, "count": 196, "max_ms": 79.424, "p95_ms": 74.043}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.295, "count": 196, "max_ms": 262.132, "p95_ms": 136.787}, "read_interval": {"avg_ms": 130.938, "count": 195, "max_ms": 274.027, "p95_ms": 149.557}, "triton_wait": {"avg_ms": 57.123, "count": 196, "max_ms": 79.436, "p95_ms": 74.054}, "visualize": {"avg_ms": 4.71, "count": 196, "max_ms": 7.044, "p95_ms": 5.074}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16239.426, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56109641f640] error while decoding MB 67 4, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 780, "h264_decode_error_count": 1914, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:18:35+09:00", "last_error_summary": "[h264 @ 0x56109641f640] error while decoding MB 67 4, bytestream -10", "last_exit_at": "2026-07-06T05:14:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 225.151, "last_started_at": "2026-07-06T05:14:50+09:00", "pid": 145662, "process": {"exists": false, "pid": 145662}, "restart_count": 868, "restart_count_per_hour": 7, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 869}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.59, "sample_count": 2274, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1909840, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:18:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T05:18:35+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T05:18:35+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T05:18:35.130537+09:00", "mtime_epoch": 1783282715.130537, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:18:35+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9536.621, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 232, "pushed_frames": 232, "read_frames": 232, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.765, "pushed": 7.765, "read": 7.765}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.927, "count": 232, "max_ms": 29.205, "p95_ms": 20.994}, "gstreamer_push": {"avg_ms": 1.114, "count": 232, "max_ms": 1.335, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.733, "count": 232, "max_ms": 66.422, "p95_ms": 61.478}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.717, "count": 232, "max_ms": 78.954, "p95_ms": 68.672}, "yolo11_640_fire_11class": {"avg_ms": 47.879, "count": 232, "max_ms": 65.258, "p95_ms": 60.72}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.226, "count": 232, "max_ms": 78.08, "p95_ms": 74.467}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.184, "count": 232, "max_ms": 140.402, "p95_ms": 135.178}, "read_interval": {"avg_ms": 129.118, "count": 232, "max_ms": 154.628, "p95_ms": 148.887}, "triton_wait": {"avg_ms": 55.672, "count": 232, "max_ms": 78.965, "p95_ms": 74.476}, "visualize": {"avg_ms": 4.678, "count": 232, "max_ms": 6.125, "p95_ms": 4.808}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16238.984, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:18:35+09:00", "last_error_summary": "[h264 @ 0x56109641f640] error while decoding MB 67 4, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 780, "h264_decode_error_count": 1914, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:18:35+09:00", "last_error_summary": "[h264 @ 0x56109641f640] error while decoding MB 67 4, bytestream -10", "last_exit_at": "2026-07-06T05:18:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 225.237, "last_started_at": "2026-07-06T05:14:50+09:00", "pid": 145662, "process": {"exists": false, "pid": 145662}, "restart_count": 868, "restart_count_per_hour": 7, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 869}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.59, "sample_count": 2274, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1909841, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:18:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T05:18:35+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-06T05:18:35+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-06T05:18:35.130537+09:00", "mtime_epoch": 1783282715.130537, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:18:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9536.621, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 233, "pushed_frames": 233, "read_frames": 232, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.776, "pushed": 7.776, "read": 7.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.953, "count": 233, "max_ms": 29.205, "p95_ms": 21.046}, "gstreamer_push": {"avg_ms": 1.114, "count": 233, "max_ms": 1.335, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.72, "count": 233, "max_ms": 66.422, "p95_ms": 61.478}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.733, "count": 233, "max_ms": 78.954, "p95_ms": 68.672}, "yolo11_640_fire_11class": {"avg_ms": 47.867, "count": 233, "max_ms": 65.258, "p95_ms": 60.72}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.229, "count": 233, "max_ms": 78.08, "p95_ms": 74.467}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.241, "count": 233, "max_ms": 140.402, "p95_ms": 135.178}, "read_interval": {"avg_ms": 129.118, "count": 232, "max_ms": 154.628, "p95_ms": 148.887}, "triton_wait": {"avg_ms": 55.675, "count": 233, "max_ms": 78.965, "p95_ms": 74.476}, "visualize": {"avg_ms": 4.685, "count": 233, "max_ms": 6.201, "p95_ms": 4.811}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16239.102, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 780, "h264_decode_error_count": 1914, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:18:35+09:00", "last_error_summary": "[h264 @ 0x56109641f640] error while decoding MB 67 4, bytestream -10", "last_exit_at": "2026-07-06T05:18:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 225.244, "last_started_at": "2026-07-06T05:14:50+09:00", "pid": 145662, "process": {"exists": false, "pid": 145662}, "restart_count": 868, "restart_count_per_hour": 7, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 869}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.59, "sample_count": 2274, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1909841, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:18:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T05:18:35+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T05:18:35+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-06T05:18:35.130537+09:00", "mtime_epoch": 1783282715.130537, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:18:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9536.621, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 233, "pushed_frames": 233, "read_frames": 232, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.775, "pushed": 7.775, "read": 7.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.954, "count": 232, "max_ms": 29.205, "p95_ms": 21.046}, "gstreamer_push": {"avg_ms": 1.114, "count": 233, "max_ms": 1.335, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.681, "count": 232, "max_ms": 66.422, "p95_ms": 61.478}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.68, "count": 232, "max_ms": 78.954, "p95_ms": 68.672}, "yolo11_640_fire_11class": {"avg_ms": 47.831, "count": 232, "max_ms": 65.258, "p95_ms": 60.72}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.17, "count": 232, "max_ms": 78.08, "p95_ms": 74.467}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.241, "count": 233, "max_ms": 140.402, "p95_ms": 135.178}, "read_interval": {"avg_ms": 129.118, "count": 232, "max_ms": 154.628, "p95_ms": 148.887}, "triton_wait": {"avg_ms": 55.618, "count": 232, "max_ms": 78.965, "p95_ms": 74.476}, "visualize": {"avg_ms": 4.684, "count": 232, "max_ms": 6.201, "p95_ms": 4.811}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16344.512, "limit_mb": 8796093022207.996}, "detail": {"pid": 145769, "restart_count": 869, "start_count": 870}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 780, "h264_decode_error_count": 1914, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:18:35+09:00", "last_error_summary": "[h264 @ 0x56109641f640] error while decoding MB 67 4, bytestream -10", "last_exit_at": "2026-07-06T05:18:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:18:37+09:00", "pid": 145769, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.801, "name": "ffmpeg", "pid": 145769, "rss_mb": 136.801, "state": "S (sleeping)", "threads": 106}, "restart_count": 869, "restart_count_per_hour": 8, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 870}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.577, "sample_count": 2239, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1909841, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:18:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.699, "last_frame_pushed_at": "2026-07-06T05:18:35+09:00", "last_frame_read_age_sec": 4.83, "last_frame_read_at": "2026-07-06T05:18:35+09:00", "mjpeg": {"age_sec": 4.68, "exists": true, "mtime": "2026-07-06T05:18:35.274533+09:00", "mtime_epoch": 1783282715.2745333, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182157}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:18:39+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9536.621, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 195, "pushed_frames": 195, "read_frames": 194, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.517, "pushed": 6.517, "read": 6.483}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.013, "count": 195, "max_ms": 29.205, "p95_ms": 21.062}, "gstreamer_push": {"avg_ms": 1.122, "count": 195, "max_ms": 1.335, "p95_ms": 1.197}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.1, "count": 195, "max_ms": 66.422, "p95_ms": 61.95}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.976, "count": 195, "max_ms": 78.954, "p95_ms": 68.818}, "yolo11_640_fire_11class": {"avg_ms": 48.23, "count": 195, "max_ms": 65.258, "p95_ms": 61.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.836, "count": 195, "max_ms": 78.08, "p95_ms": 74.975}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.96, "count": 195, "max_ms": 140.402, "p95_ms": 136.259}, "read_interval": {"avg_ms": 129.904, "count": 194, "max_ms": 154.628, "p95_ms": 149.193}, "triton_wait": {"avg_ms": 56.273, "count": 195, "max_ms": 78.965, "p95_ms": 74.987}, "visualize": {"avg_ms": 4.685, "count": 195, "max_ms": 6.201, "p95_ms": 4.811}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15928}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16241.098, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555e09db5f00] error while decoding MB 72 9, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 781, "h264_decode_error_count": 1915, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:22:06+09:00", "last_error_summary": "[h264 @ 0x555e09db5f00] error while decoding MB 72 9, bytestream -5", "last_exit_at": "2026-07-06T05:18:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 208.672, "last_started_at": "2026-07-06T05:18:37+09:00", "pid": 145769, "process": {"exists": false, "pid": 145769}, "restart_count": 869, "restart_count_per_hour": 8, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 870}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.46, "sample_count": 2248, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1911404, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:22:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-06T05:22:06+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-06T05:22:06+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-06T05:22:06.562626+09:00", "mtime_epoch": 1783282926.562626, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155460}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:22:06+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9537.395, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 225, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.516, "pushed": 7.516, "read": 7.516}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.307, "count": 225, "max_ms": 26.429, "p95_ms": 21.587}, "gstreamer_push": {"avg_ms": 1.13, "count": 225, "max_ms": 1.565, "p95_ms": 1.236}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.921, "count": 225, "max_ms": 65.533, "p95_ms": 62.325}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.108, "count": 225, "max_ms": 75.962, "p95_ms": 67.111}, "yolo11_640_fire_11class": {"avg_ms": 49.43, "count": 225, "max_ms": 66.102, "p95_ms": 61.899}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.725, "count": 225, "max_ms": 80.394, "p95_ms": 76.344}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.591, "count": 225, "max_ms": 174.564, "p95_ms": 160.841}, "read_interval": {"avg_ms": 133.29, "count": 225, "max_ms": 186.545, "p95_ms": 172.328}, "triton_wait": {"avg_ms": 57.795, "count": 225, "max_ms": 80.404, "p95_ms": 76.355}, "visualize": {"avg_ms": 4.774, "count": 225, "max_ms": 6.174, "p95_ms": 5.832}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15928}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16240.836, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:22:06+09:00", "last_error_summary": "[h264 @ 0x555e09db5f00] error while decoding MB 72 9, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 781, "h264_decode_error_count": 1915, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:22:06+09:00", "last_error_summary": "[h264 @ 0x555e09db5f00] error while decoding MB 72 9, bytestream -5", "last_exit_at": "2026-07-06T05:22:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 208.737, "last_started_at": "2026-07-06T05:18:37+09:00", "pid": 145769, "process": {"exists": false, "pid": 145769}, "restart_count": 869, "restart_count_per_hour": 8, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 870}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.46, "sample_count": 2248, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1911405, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:22:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T05:22:06+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-06T05:22:06+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-06T05:22:06.562626+09:00", "mtime_epoch": 1783282926.562626, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155460}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:22:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9537.395, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 226, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.5, "pushed": 7.533, "read": 7.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.33, "count": 225, "max_ms": 26.429, "p95_ms": 21.916}, "gstreamer_push": {"avg_ms": 1.13, "count": 226, "max_ms": 1.565, "p95_ms": 1.236}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.902, "count": 225, "max_ms": 65.533, "p95_ms": 62.325}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.066, "count": 225, "max_ms": 75.962, "p95_ms": 67.111}, "yolo11_640_fire_11class": {"avg_ms": 49.402, "count": 225, "max_ms": 66.102, "p95_ms": 61.899}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.672, "count": 225, "max_ms": 80.394, "p95_ms": 76.344}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.605, "count": 225, "max_ms": 174.564, "p95_ms": 160.841}, "read_interval": {"avg_ms": 133.29, "count": 225, "max_ms": 186.545, "p95_ms": 172.328}, "triton_wait": {"avg_ms": 57.742, "count": 225, "max_ms": 80.404, "p95_ms": 76.355}, "visualize": {"avg_ms": 4.785, "count": 225, "max_ms": 6.97, "p95_ms": 5.847}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15928}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16240.809, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 781, "h264_decode_error_count": 1915, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:22:06+09:00", "last_error_summary": "[h264 @ 0x555e09db5f00] error while decoding MB 72 9, bytestream -5", "last_exit_at": "2026-07-06T05:22:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 208.744, "last_started_at": "2026-07-06T05:18:37+09:00", "pid": 145769, "process": {"exists": false, "pid": 145769}, "restart_count": 869, "restart_count_per_hour": 8, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 870}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.46, "sample_count": 2248, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1911405, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:22:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T05:22:06+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T05:22:06+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-06T05:22:06.562626+09:00", "mtime_epoch": 1783282926.562626, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155460}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:22:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9537.395, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 225, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.502, "pushed": 7.502, "read": 7.502}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.33, "count": 225, "max_ms": 26.429, "p95_ms": 21.916}, "gstreamer_push": {"avg_ms": 1.13, "count": 225, "max_ms": 1.565, "p95_ms": 1.236}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.902, "count": 225, "max_ms": 65.533, "p95_ms": 62.325}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.066, "count": 225, "max_ms": 75.962, "p95_ms": 67.111}, "yolo11_640_fire_11class": {"avg_ms": 49.402, "count": 225, "max_ms": 66.102, "p95_ms": 61.899}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.672, "count": 225, "max_ms": 80.394, "p95_ms": 76.344}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.605, "count": 225, "max_ms": 174.564, "p95_ms": 160.841}, "read_interval": {"avg_ms": 133.29, "count": 225, "max_ms": 186.545, "p95_ms": 172.328}, "triton_wait": {"avg_ms": 57.742, "count": 225, "max_ms": 80.404, "p95_ms": 76.355}, "visualize": {"avg_ms": 4.785, "count": 225, "max_ms": 6.97, "p95_ms": 5.847}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15928}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16349.031, "limit_mb": 8796093022207.996}, "detail": {"pid": 145876, "restart_count": 870, "start_count": 871}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 781, "h264_decode_error_count": 1915, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:22:06+09:00", "last_error_summary": "[h264 @ 0x555e09db5f00] error while decoding MB 72 9, bytestream -5", "last_exit_at": "2026-07-06T05:22:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:22:09+09:00", "pid": 145876, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.98, "name": "ffmpeg", "pid": 145876, "rss_mb": 138.98, "state": "S (sleeping)", "threads": 106}, "restart_count": 870, "restart_count_per_hour": 9, "rtsp_timeout_count": 492, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 871}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.489, "sample_count": 2213, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1911405, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:22:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.651, "last_frame_pushed_at": "2026-07-06T05:22:06+09:00", "last_frame_read_age_sec": 4.778, "last_frame_read_at": "2026-07-06T05:22:06+09:00", "mjpeg": {"age_sec": 4.634, "exists": true, "mtime": "2026-07-06T05:22:06.702624+09:00", "mtime_epoch": 1783282926.702624, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157851}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:22:11+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9537.395, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.313, "pushed": 6.313, "read": 6.28}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.422, "count": 189, "max_ms": 26.429, "p95_ms": 22.322}, "gstreamer_push": {"avg_ms": 1.134, "count": 189, "max_ms": 1.565, "p95_ms": 1.244}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.216, "count": 189, "max_ms": 65.533, "p95_ms": 62.376}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.084, "count": 189, "max_ms": 75.962, "p95_ms": 68.911}, "yolo11_640_fire_11class": {"avg_ms": 49.658, "count": 189, "max_ms": 66.102, "p95_ms": 62.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.127, "count": 189, "max_ms": 80.394, "p95_ms": 76.537}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.647, "count": 189, "max_ms": 174.564, "p95_ms": 161.379}, "read_interval": {"avg_ms": 134.411, "count": 188, "max_ms": 186.545, "p95_ms": 173.373}, "triton_wait": {"avg_ms": 58.204, "count": 189, "max_ms": 80.404, "p95_ms": 76.549}, "visualize": {"avg_ms": 4.809, "count": 189, "max_ms": 6.97, "p95_ms": 5.869}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16242.961, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5583ecd4af40] left block unavailable for requested intra mode [h264 @ 0x5583ecd4af40] error while decoding MB 0 10, bytestream 4851 [in#0/rtsp @ 0x5583ec398e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583ec398e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 782, "h264_decode_error_count": 1916, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:25:22+09:00", "last_error_summary": "[h264 @ 0x5583ecd4af40] left block unavailable for requested intra mode [h264 @ 0x5583ecd4af40] error while decoding MB 0 10, bytestream 4851 [in#0/rtsp @ 0x5583ec398e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583ec398e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T05:22:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.454, "last_started_at": "2026-07-06T05:22:09+09:00", "pid": 145876, "process": {"exists": false, "pid": 145876}, "restart_count": 870, "restart_count_per_hour": 9, "rtsp_timeout_count": 494, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 871}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.71, "sample_count": 2114, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1912739, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T05:25:22+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T05:25:22+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-06T05:25:22.745057+09:00", "mtime_epoch": 1783283122.7450569, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166240}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:25:22+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9538.168, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 87, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.867, "pushed": 2.867, "read": 2.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.755, "count": 86, "max_ms": 33.783, "p95_ms": 21.674}, "gstreamer_push": {"avg_ms": 1.089, "count": 86, "max_ms": 1.331, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.557, "count": 86, "max_ms": 63.841, "p95_ms": 61.035}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.36, "count": 86, "max_ms": 78.181, "p95_ms": 69.644}, "yolo11_640_fire_11class": {"avg_ms": 48.73, "count": 86, "max_ms": 62.571, "p95_ms": 60.884}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.688, "count": 86, "max_ms": 76.688, "p95_ms": 72.533}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.574, "count": 86, "max_ms": 144.343, "p95_ms": 140.588}, "read_interval": {"avg_ms": 345.627, "count": 87, "max_ms": 10132.604, "p95_ms": 155.236}, "triton_wait": {"avg_ms": 56.407, "count": 86, "max_ms": 78.193, "p95_ms": 73.61}, "visualize": {"avg_ms": 4.553, "count": 86, "max_ms": 5.567, "p95_ms": 5.435}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16242.543, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:25:22+09:00", "last_error_summary": "[h264 @ 0x5583ecd4af40] left block unavailable for requested intra mode [h264 @ 0x5583ecd4af40] error while decoding MB 0 10, bytestream 4851 [in#0/rtsp @ 0x5583ec398e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583ec398e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 782, "h264_decode_error_count": 1916, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:25:22+09:00", "last_error_summary": "[h264 @ 0x5583ecd4af40] left block unavailable for requested intra mode [h264 @ 0x5583ecd4af40] error while decoding MB 0 10, bytestream 4851 [in#0/rtsp @ 0x5583ec398e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583ec398e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T05:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.535, "last_started_at": "2026-07-06T05:22:09+09:00", "pid": 145876, "process": {"exists": false, "pid": 145876}, "restart_count": 870, "restart_count_per_hour": 9, "rtsp_timeout_count": 494, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 871}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.71, "sample_count": 2115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1912740, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T05:25:22+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T05:25:22+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-06T05:25:22.745057+09:00", "mtime_epoch": 1783283122.7450569, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166240}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:25:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9538.168, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.905, "pushed": 2.905, "read": 2.872}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.749, "count": 87, "max_ms": 33.783, "p95_ms": 21.674}, "gstreamer_push": {"avg_ms": 1.089, "count": 87, "max_ms": 1.331, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.681, "count": 87, "max_ms": 63.841, "p95_ms": 61.035}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.538, "count": 87, "max_ms": 78.181, "p95_ms": 69.644}, "yolo11_640_fire_11class": {"avg_ms": 48.814, "count": 87, "max_ms": 62.571, "p95_ms": 60.884}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.831, "count": 87, "max_ms": 76.688, "p95_ms": 72.533}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.704, "count": 87, "max_ms": 144.343, "p95_ms": 140.588}, "read_interval": {"avg_ms": 348.196, "count": 86, "max_ms": 10132.604, "p95_ms": 155.236}, "triton_wait": {"avg_ms": 56.549, "count": 87, "max_ms": 78.193, "p95_ms": 73.61}, "visualize": {"avg_ms": 4.555, "count": 87, "max_ms": 5.567, "p95_ms": 5.435}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16242.535, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 782, "h264_decode_error_count": 1916, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:25:22+09:00", "last_error_summary": "[h264 @ 0x5583ecd4af40] left block unavailable for requested intra mode [h264 @ 0x5583ecd4af40] error while decoding MB 0 10, bytestream 4851 [in#0/rtsp @ 0x5583ec398e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583ec398e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T05:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.54, "last_started_at": "2026-07-06T05:22:09+09:00", "pid": 145876, "process": {"exists": false, "pid": 145876}, "restart_count": 870, "restart_count_per_hour": 9, "rtsp_timeout_count": 494, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 871}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.71, "sample_count": 2115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1912740, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T05:25:22+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-06T05:25:22+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T05:25:22.745057+09:00", "mtime_epoch": 1783283122.7450569, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166240}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:25:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9538.168, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.905, "pushed": 2.905, "read": 2.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.749, "count": 87, "max_ms": 33.783, "p95_ms": 21.674}, "gstreamer_push": {"avg_ms": 1.089, "count": 87, "max_ms": 1.331, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.681, "count": 87, "max_ms": 63.841, "p95_ms": 61.035}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.538, "count": 87, "max_ms": 78.181, "p95_ms": 69.644}, "yolo11_640_fire_11class": {"avg_ms": 48.814, "count": 87, "max_ms": 62.571, "p95_ms": 60.884}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.831, "count": 87, "max_ms": 76.688, "p95_ms": 72.533}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.704, "count": 87, "max_ms": 144.343, "p95_ms": 140.588}, "read_interval": {"avg_ms": 348.196, "count": 86, "max_ms": 10132.604, "p95_ms": 155.236}, "triton_wait": {"avg_ms": 56.549, "count": 87, "max_ms": 78.193, "p95_ms": 73.61}, "visualize": {"avg_ms": 4.555, "count": 87, "max_ms": 5.567, "p95_ms": 5.435}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16350.953, "limit_mb": 8796093022207.996}, "detail": {"pid": 145983, "restart_count": 871, "start_count": 872}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 782, "h264_decode_error_count": 1916, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:25:22+09:00", "last_error_summary": "[h264 @ 0x5583ecd4af40] left block unavailable for requested intra mode [h264 @ 0x5583ecd4af40] error while decoding MB 0 10, bytestream 4851 [in#0/rtsp @ 0x5583ec398e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583ec398e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T05:25:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:25:25+09:00", "pid": 145983, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.852, "name": "ffmpeg", "pid": 145983, "rss_mb": 138.852, "state": "S (sleeping)", "threads": 106}, "restart_count": 871, "restart_count_per_hour": 10, "rtsp_timeout_count": 494, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 872}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.71, "sample_count": 2081, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1912740, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.488, "last_frame_pushed_at": "2026-07-06T05:25:22+09:00", "last_frame_read_age_sec": 4.616, "last_frame_read_at": "2026-07-06T05:25:22+09:00", "mjpeg": {"age_sec": 4.472, "exists": true, "mtime": "2026-07-06T05:25:22.881056+09:00", "mtime_epoch": 1783283122.8810563, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166318}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:25:27+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9538.168, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 52, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.734, "pushed": 1.734, "read": 1.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.775, "count": 52, "max_ms": 33.783, "p95_ms": 31.697}, "gstreamer_push": {"avg_ms": 1.071, "count": 52, "max_ms": 1.331, "p95_ms": 1.211}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.741, "count": 52, "max_ms": 61.792, "p95_ms": 60.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.285, "count": 52, "max_ms": 78.181, "p95_ms": 68.804}, "yolo11_640_fire_11class": {"avg_ms": 48.923, "count": 52, "max_ms": 62.329, "p95_ms": 60.286}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.503, "count": 52, "max_ms": 75.434, "p95_ms": 71.885}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.544, "count": 52, "max_ms": 144.343, "p95_ms": 142.74}, "read_interval": {"avg_ms": 491.059, "count": 52, "max_ms": 10132.604, "p95_ms": 156.685}, "triton_wait": {"avg_ms": 56.128, "count": 52, "max_ms": 78.193, "p95_ms": 71.898}, "visualize": {"avg_ms": 4.453, "count": 52, "max_ms": 5.567, "p95_ms": 5.497}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 19788}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16486.773, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 140, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-05T16:11:35+09:00", "last_error_summary": "[h264 @ 0x55eadc83c280] error while decoding MB 26 45, bytestream -23 [in#0/rtsp @ 0x55eadc694e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55eadc694e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-05T16:11:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 47693.344, "last_started_at": "2026-07-05T16:11:38+09:00", "pid": 136436, "process": {"exists": true, "fd_count": 5, "hwm_mb": 332.801, "name": "ffmpeg", "pid": 136436, "rss_mb": 332.801, "state": "S (sleeping)", "threads": 106}, "restart_count": 141, "restart_count_per_hour": 0, "rtsp_timeout_count": 240, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 90.565, "sample_count": 2697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2374629, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:26:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 30.059, "last_frame_pushed_at": "2026-07-06T05:26:01+09:00", "last_frame_read_age_sec": 30.126, "last_frame_read_at": "2026-07-06T05:26:01+09:00", "mjpeg": {"age_sec": 30.042, "exists": true, "mtime": "2026-07-06T05:26:01.504921+09:00", "mtime_epoch": 1783283161.504921, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 181762}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T05:26:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2574.086, "name": "celery", "pid": 267, "rss_mb": 2574.086, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 19788}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16466.859, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 140, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-05T16:11:35+09:00", "last_error_summary": "[h264 @ 0x55eadc83c280] error while decoding MB 26 45, bytestream -23 [in#0/rtsp @ 0x55eadc694e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55eadc694e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-05T16:11:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 47693.357, "last_started_at": "2026-07-05T16:11:38+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 141, "restart_count_per_hour": 0, "rtsp_timeout_count": 240, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 90.565, "sample_count": 2697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2374629, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:26:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 30.072, "last_frame_pushed_at": "2026-07-06T05:26:01+09:00", "last_frame_read_age_sec": 30.139, "last_frame_read_at": "2026-07-06T05:26:01+09:00", "mjpeg": {"age_sec": 30.055, "exists": true, "mtime": "2026-07-06T05:26:01.504921+09:00", "mtime_epoch": 1783283161.504921, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 181762}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T05:26:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2574.086, "name": "celery", "pid": 267, "rss_mb": 2574.086, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 19788}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16349.422, "limit_mb": 8796093022207.996}, "detail": {"pid": 146090, "restart_count": 142, "start_count": 143}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 140, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-05T16:11:35+09:00", "last_error_summary": "[h264 @ 0x55eadc83c280] error while decoding MB 26 45, bytestream -23 [in#0/rtsp @ 0x55eadc694e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55eadc694e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-05T16:11:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:26:33+09:00", "pid": 146090, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.559, "name": "ffmpeg", "pid": 146090, "rss_mb": 197.559, "state": "S (sleeping)", "threads": 106}, "restart_count": 142, "restart_count_per_hour": 1, "rtsp_timeout_count": 240, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 90.561, "sample_count": 2654, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2374629, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:26:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 34.288, "last_frame_pushed_at": "2026-07-06T05:26:01+09:00", "last_frame_read_age_sec": 34.355, "last_frame_read_at": "2026-07-06T05:26:01+09:00", "mjpeg": {"age_sec": 34.271, "exists": true, "mtime": "2026-07-06T05:26:01.504921+09:00", "mtime_epoch": 1783283161.504921, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 181762}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T05:26:35+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2574.086, "name": "celery", "pid": 267, "rss_mb": 2574.086, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15949}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16242.945, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b060487580] left block unavailable for requested intra mode [h264 @ 0x55b060487580] error while decoding MB 0 23, bytestream 116820 [h264 @ 0x55b0605b8bc0] cabac decode of qscale diff failed at 76 40 [h264 @ 0x55b0605b8bc0] error while decoding MB 76 40, bytestream 3285 [h264 @ 0x55b06123e200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b06123e200] error while decoding MB 0 42, bytestream 293 [h264 @ 0x55b06056f440] left block unavailable for requested intra4x4 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 783, "h264_decode_error_count": 1921, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:28:18+09:00", "last_error_summary": "[h264 @ 0x55b060487580] left block unavailable for requested intra mode [h264 @ 0x55b060487580] error while decoding MB 0 23, bytestream 116820 [h264 @ 0x55b0605b8bc0] cabac decode of qscale diff failed at 76 40 [h264 @ 0x55b0605b8bc0] error while decoding MB 76 40, bytestream 3285 [h264 @ 0x55b06123e200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b06123e200] error while decoding MB 0 42, bytestream 293 [h264 @ 0x55b06056f440] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-06T05:25:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 172.97, "last_started_at": "2026-07-06T05:25:25+09:00", "pid": 145983, "process": {"exists": false, "pid": 145983}, "restart_count": 871, "restart_count_per_hour": 10, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 872}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.204, "sample_count": 2016, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1913939, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:28:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-06T05:28:18+09:00", "last_frame_read_age_sec": 0.073, "last_frame_read_at": "2026-07-06T05:28:18+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-06T05:28:18.248800+09:00", "mtime_epoch": 1783283298.2488, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:28:18+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9538.684, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 86, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.876, "pushed": 2.876, "read": 2.876}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.486, "count": 86, "max_ms": 21.636, "p95_ms": 20.55}, "gstreamer_push": {"avg_ms": 1.143, "count": 86, "max_ms": 1.455, "p95_ms": 1.288}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.607, "count": 86, "max_ms": 63.493, "p95_ms": 62.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.212, "count": 86, "max_ms": 79.641, "p95_ms": 73.136}, "yolo11_640_fire_11class": {"avg_ms": 49.976, "count": 86, "max_ms": 64.639, "p95_ms": 61.888}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.539, "count": 86, "max_ms": 79.498, "p95_ms": 75.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.758, "count": 86, "max_ms": 159.735, "p95_ms": 139.143}, "read_interval": {"avg_ms": 348.414, "count": 86, "max_ms": 19254.295, "p95_ms": 152.038}, "triton_wait": {"avg_ms": 58.817, "count": 86, "max_ms": 79.652, "p95_ms": 75.956}, "visualize": {"avg_ms": 4.493, "count": 86, "max_ms": 5.906, "p95_ms": 5.74}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15949}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16242.781, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:28:18+09:00", "last_error_summary": "[h264 @ 0x55b060487580] left block unavailable for requested intra mode [h264 @ 0x55b060487580] error while decoding MB 0 23, bytestream 116820 [h264 @ 0x55b0605b8bc0] cabac decode of qscale diff failed at 76 40 [h264 @ 0x55b0605b8bc0] error while decoding MB 76 40, bytestream 3285 [h264 @ 0x55b06123e200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b06123e200] error while decoding MB 0 42, bytestream 293 [h264 @ 0x55b06056f440] left block unavailable for requested intra4x4 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 783, "h264_decode_error_count": 1921, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:28:18+09:00", "last_error_summary": "[h264 @ 0x55b060487580] left block unavailable for requested intra mode [h264 @ 0x55b060487580] error while decoding MB 0 23, bytestream 116820 [h264 @ 0x55b0605b8bc0] cabac decode of qscale diff failed at 76 40 [h264 @ 0x55b0605b8bc0] error while decoding MB 76 40, bytestream 3285 [h264 @ 0x55b06123e200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b06123e200] error while decoding MB 0 42, bytestream 293 [h264 @ 0x55b06056f440] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-06T05:28:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 173.042, "last_started_at": "2026-07-06T05:25:25+09:00", "pid": 145983, "process": {"exists": false, "pid": 145983}, "restart_count": 871, "restart_count_per_hour": 10, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 872}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.204, "sample_count": 2017, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1913940, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:28:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T05:28:18+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T05:28:18+09:00", "mjpeg": {"age_sec": 0.145, "exists": true, "mtime": "2026-07-06T05:28:18.248800+09:00", "mtime_epoch": 1783283298.2488, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:28:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9538.684, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.902, "pushed": 2.902, "read": 2.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.528, "count": 86, "max_ms": 23.136, "p95_ms": 21.097}, "gstreamer_push": {"avg_ms": 1.145, "count": 87, "max_ms": 1.455, "p95_ms": 1.288}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.433, "count": 86, "max_ms": 63.493, "p95_ms": 62.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.231, "count": 86, "max_ms": 79.641, "p95_ms": 73.136}, "yolo11_640_fire_11class": {"avg_ms": 49.826, "count": 86, "max_ms": 64.639, "p95_ms": 61.888}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.36, "count": 86, "max_ms": 79.498, "p95_ms": 75.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.037, "count": 87, "max_ms": 159.735, "p95_ms": 139.143}, "read_interval": {"avg_ms": 348.414, "count": 86, "max_ms": 19254.295, "p95_ms": 152.038}, "triton_wait": {"avg_ms": 58.638, "count": 86, "max_ms": 79.652, "p95_ms": 75.956}, "visualize": {"avg_ms": 4.53, "count": 86, "max_ms": 7.687, "p95_ms": 5.822}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15949}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16242.77, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 783, "h264_decode_error_count": 1921, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:28:18+09:00", "last_error_summary": "[h264 @ 0x55b060487580] left block unavailable for requested intra mode [h264 @ 0x55b060487580] error while decoding MB 0 23, bytestream 116820 [h264 @ 0x55b0605b8bc0] cabac decode of qscale diff failed at 76 40 [h264 @ 0x55b0605b8bc0] error while decoding MB 76 40, bytestream 3285 [h264 @ 0x55b06123e200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b06123e200] error while decoding MB 0 42, bytestream 293 [h264 @ 0x55b06056f440] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-06T05:28:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 173.048, "last_started_at": "2026-07-06T05:25:25+09:00", "pid": 145983, "process": {"exists": false, "pid": 145983}, "restart_count": 871, "restart_count_per_hour": 10, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 872}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.204, "sample_count": 2017, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1913940, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:28:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T05:28:18+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T05:28:18+09:00", "mjpeg": {"age_sec": 0.152, "exists": true, "mtime": "2026-07-06T05:28:18.248800+09:00", "mtime_epoch": 1783283298.2488, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:28:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9538.684, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.902, "pushed": 2.902, "read": 2.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.528, "count": 86, "max_ms": 23.136, "p95_ms": 21.097}, "gstreamer_push": {"avg_ms": 1.145, "count": 87, "max_ms": 1.455, "p95_ms": 1.288}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.433, "count": 86, "max_ms": 63.493, "p95_ms": 62.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.231, "count": 86, "max_ms": 79.641, "p95_ms": 73.136}, "yolo11_640_fire_11class": {"avg_ms": 49.826, "count": 86, "max_ms": 64.639, "p95_ms": 61.888}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.36, "count": 86, "max_ms": 79.498, "p95_ms": 75.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.037, "count": 87, "max_ms": 159.735, "p95_ms": 139.143}, "read_interval": {"avg_ms": 348.414, "count": 86, "max_ms": 19254.295, "p95_ms": 152.038}, "triton_wait": {"avg_ms": 58.638, "count": 86, "max_ms": 79.652, "p95_ms": 75.956}, "visualize": {"avg_ms": 4.53, "count": 86, "max_ms": 7.687, "p95_ms": 5.822}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 330, "tracking_summary": 15949}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16350.926, "limit_mb": 8796093022207.996}, "detail": {"pid": 146242, "restart_count": 872, "start_count": 873}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 783, "h264_decode_error_count": 1921, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:28:18+09:00", "last_error_summary": "[h264 @ 0x55b060487580] left block unavailable for requested intra mode [h264 @ 0x55b060487580] error while decoding MB 0 23, bytestream 116820 [h264 @ 0x55b0605b8bc0] cabac decode of qscale diff failed at 76 40 [h264 @ 0x55b0605b8bc0] error while decoding MB 76 40, bytestream 3285 [h264 @ 0x55b06123e200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b06123e200] error while decoding MB 0 42, bytestream 293 [h264 @ 0x55b06056f440] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-06T05:28:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:28:21+09:00", "pid": 146242, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.242, "name": "ffmpeg", "pid": 146242, "rss_mb": 139.242, "state": "S (sleeping)", "threads": 106}, "restart_count": 872, "restart_count_per_hour": 11, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 873}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.204, "sample_count": 1980, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1913940, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:28:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.745, "last_frame_pushed_at": "2026-07-06T05:28:18+09:00", "last_frame_read_age_sec": 4.884, "last_frame_read_at": "2026-07-06T05:28:18+09:00", "mjpeg": {"age_sec": 4.729, "exists": true, "mtime": "2026-07-06T05:28:18.404800+09:00", "mtime_epoch": 1783283298.4048002, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171454}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:28:23+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9538.684, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.738, "pushed": 1.738, "read": 1.705}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.334, "count": 52, "max_ms": 23.136, "p95_ms": 21.097}, "gstreamer_push": {"avg_ms": 1.176, "count": 52, "max_ms": 1.455, "p95_ms": 1.298}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.082, "count": 52, "max_ms": 63.493, "p95_ms": 62.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.772, "count": 52, "max_ms": 76.888, "p95_ms": 69.112}, "yolo11_640_fire_11class": {"avg_ms": 49.323, "count": 52, "max_ms": 63.491, "p95_ms": 59.394}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.877, "count": 52, "max_ms": 76.876, "p95_ms": 73.592}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.697, "count": 52, "max_ms": 137.045, "p95_ms": 134.313}, "read_interval": {"avg_ms": 492.915, "count": 51, "max_ms": 19254.295, "p95_ms": 147.993}, "triton_wait": {"avg_ms": 57.097, "count": 52, "max_ms": 76.897, "p95_ms": 73.605}, "visualize": {"avg_ms": 4.3, "count": 52, "max_ms": 7.687, "p95_ms": 5.262}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 332, "tracking_summary": 15961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16251.07, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a5e06256c0] error while decoding MB 65 3, bytestream -62", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 784, "h264_decode_error_count": 1922, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:31:36+09:00", "last_error_summary": "[h264 @ 0x55a5e06256c0] error while decoding MB 65 3, bytestream -62", "last_exit_at": "2026-07-06T05:28:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.693, "last_started_at": "2026-07-06T05:28:21+09:00", "pid": 146242, "process": {"exists": false, "pid": 146242}, "restart_count": 872, "restart_count_per_hour": 11, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 873}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.968, "sample_count": 2056, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1915350, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:31:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T05:31:36+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T05:31:36+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T05:31:36.785410+09:00", "mtime_epoch": 1783283496.78541, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174657}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:31:36+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9539.457, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.867, "pushed": 6.867, "read": 6.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.937, "count": 206, "max_ms": 27.722, "p95_ms": 22.287}, "gstreamer_push": {"avg_ms": 1.075, "count": 206, "max_ms": 1.641, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.196, "count": 206, "max_ms": 63.769, "p95_ms": 61.356}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.231, "count": 206, "max_ms": 80.486, "p95_ms": 72.836}, "yolo11_640_fire_11class": {"avg_ms": 49.083, "count": 206, "max_ms": 64.851, "p95_ms": 61.911}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.084, "count": 206, "max_ms": 78.729, "p95_ms": 75.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.238, "count": 206, "max_ms": 168.755, "p95_ms": 159.809}, "read_interval": {"avg_ms": 145.417, "count": 207, "max_ms": 182.24, "p95_ms": 168.745}, "triton_wait": {"avg_ms": 58.401, "count": 206, "max_ms": 80.497, "p95_ms": 76.561}, "visualize": {"avg_ms": 5.331, "count": 206, "max_ms": 7.734, "p95_ms": 6.175}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 332, "tracking_summary": 15961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16250.395, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:31:36+09:00", "last_error_summary": "[h264 @ 0x55a5e06256c0] error while decoding MB 65 3, bytestream -62", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 784, "h264_decode_error_count": 1922, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:31:36+09:00", "last_error_summary": "[h264 @ 0x55a5e06256c0] error while decoding MB 65 3, bytestream -62", "last_exit_at": "2026-07-06T05:31:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.795, "last_started_at": "2026-07-06T05:28:21+09:00", "pid": 146242, "process": {"exists": false, "pid": 146242}, "restart_count": 872, "restart_count_per_hour": 11, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 873}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.968, "sample_count": 2056, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1915351, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:31:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T05:31:36+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-06T05:31:36+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T05:31:36.785410+09:00", "mtime_epoch": 1783283496.78541, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174657}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:31:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9539.457, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.904, "pushed": 6.904, "read": 6.87}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.953, "count": 206, "max_ms": 27.722, "p95_ms": 22.325}, "gstreamer_push": {"avg_ms": 1.075, "count": 207, "max_ms": 1.641, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.205, "count": 206, "max_ms": 63.769, "p95_ms": 61.356}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.247, "count": 206, "max_ms": 80.486, "p95_ms": 72.836}, "yolo11_640_fire_11class": {"avg_ms": 49.091, "count": 206, "max_ms": 64.851, "p95_ms": 61.911}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.091, "count": 206, "max_ms": 78.729, "p95_ms": 75.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.295, "count": 207, "max_ms": 168.755, "p95_ms": 159.809}, "read_interval": {"avg_ms": 145.379, "count": 206, "max_ms": 182.24, "p95_ms": 168.745}, "triton_wait": {"avg_ms": 58.412, "count": 206, "max_ms": 80.497, "p95_ms": 76.561}, "visualize": {"avg_ms": 5.338, "count": 206, "max_ms": 7.734, "p95_ms": 6.238}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 332, "tracking_summary": 15961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16250.387, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 784, "h264_decode_error_count": 1922, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:31:36+09:00", "last_error_summary": "[h264 @ 0x55a5e06256c0] error while decoding MB 65 3, bytestream -62", "last_exit_at": "2026-07-06T05:31:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.8, "last_started_at": "2026-07-06T05:28:21+09:00", "pid": 146242, "process": {"exists": false, "pid": 146242}, "restart_count": 872, "restart_count_per_hour": 11, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 873}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.968, "sample_count": 2056, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1915351, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:31:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T05:31:36+09:00", "last_frame_read_age_sec": 0.157, "last_frame_read_at": "2026-07-06T05:31:36+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-06T05:31:36.785410+09:00", "mtime_epoch": 1783283496.78541, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174657}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:31:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9539.457, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.903, "pushed": 6.903, "read": 6.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.953, "count": 206, "max_ms": 27.722, "p95_ms": 22.325}, "gstreamer_push": {"avg_ms": 1.075, "count": 207, "max_ms": 1.641, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.205, "count": 206, "max_ms": 63.769, "p95_ms": 61.356}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.247, "count": 206, "max_ms": 80.486, "p95_ms": 72.836}, "yolo11_640_fire_11class": {"avg_ms": 49.091, "count": 206, "max_ms": 64.851, "p95_ms": 61.911}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.091, "count": 206, "max_ms": 78.729, "p95_ms": 75.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.295, "count": 207, "max_ms": 168.755, "p95_ms": 159.809}, "read_interval": {"avg_ms": 145.379, "count": 206, "max_ms": 182.24, "p95_ms": 168.745}, "triton_wait": {"avg_ms": 58.412, "count": 206, "max_ms": 80.497, "p95_ms": 76.561}, "visualize": {"avg_ms": 5.338, "count": 206, "max_ms": 7.734, "p95_ms": 6.238}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 332, "tracking_summary": 15961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16358.402, "limit_mb": 8796093022207.996}, "detail": {"pid": 146409, "restart_count": 873, "start_count": 874}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 784, "h264_decode_error_count": 1922, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:31:36+09:00", "last_error_summary": "[h264 @ 0x55a5e06256c0] error while decoding MB 65 3, bytestream -62", "last_exit_at": "2026-07-06T05:31:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:31:39+09:00", "pid": 146409, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.145, "name": "ffmpeg", "pid": 146409, "rss_mb": 139.145, "state": "S (sleeping)", "threads": 106}, "restart_count": 873, "restart_count_per_hour": 12, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 874}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.606, "sample_count": 2020, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1915351, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:31:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.848, "last_frame_pushed_at": "2026-07-06T05:31:36+09:00", "last_frame_read_age_sec": 4.996, "last_frame_read_at": "2026-07-06T05:31:36+09:00", "mjpeg": {"age_sec": 4.83, "exists": true, "mtime": "2026-07-06T05:31:36.941411+09:00", "mtime_epoch": 1783283496.9414108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174585}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:31:41+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9539.457, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 173, "pushed_frames": 173, "read_frames": 172, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.768, "pushed": 5.768, "read": 5.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.987, "count": 172, "max_ms": 27.722, "p95_ms": 22.325}, "gstreamer_push": {"avg_ms": 1.069, "count": 173, "max_ms": 1.641, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.107, "count": 172, "max_ms": 63.769, "p95_ms": 61.356}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.881, "count": 172, "max_ms": 80.486, "p95_ms": 72.836}, "yolo11_640_fire_11class": {"avg_ms": 48.994, "count": 172, "max_ms": 63.629, "p95_ms": 61.138}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.211, "count": 172, "max_ms": 78.729, "p95_ms": 75.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.361, "count": 173, "max_ms": 168.755, "p95_ms": 160.432}, "read_interval": {"avg_ms": 146.194, "count": 172, "max_ms": 182.24, "p95_ms": 171.398}, "triton_wait": {"avg_ms": 58.295, "count": 172, "max_ms": 80.497, "p95_ms": 76.586}, "visualize": {"avg_ms": 5.371, "count": 172, "max_ms": 7.734, "p95_ms": 6.363}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 333, "tracking_summary": 15973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16258.156, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557b313420c0] left block unavailable for requested intra mode [h264 @ 0x557b313420c0] error while decoding MB 0 35, bytestream 4703 [h264 @ 0x557b31f0c8c0] error while decoding MB 74 34, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 785, "h264_decode_error_count": 1924, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:35:00+09:00", "last_error_summary": "[h264 @ 0x557b313420c0] left block unavailable for requested intra mode [h264 @ 0x557b313420c0] error while decoding MB 0 35, bytestream 4703 [h264 @ 0x557b31f0c8c0] error while decoding MB 74 34, bytestream -6", "last_exit_at": "2026-07-06T05:31:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 200.243, "last_started_at": "2026-07-06T05:31:39+09:00", "pid": 146409, "process": {"exists": false, "pid": 146409}, "restart_count": 873, "restart_count_per_hour": 12, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 874}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.676, "sample_count": 2111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1916780, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:34:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T05:34:59+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T05:34:59+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T05:34:59.970715+09:00", "mtime_epoch": 1783283699.9707146, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168975}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:35:00+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9539.973, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 224, "pushed_frames": 224, "read_frames": 224, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.499, "pushed": 7.499, "read": 7.499}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.383, "count": 224, "max_ms": 26.44, "p95_ms": 21.097}, "gstreamer_push": {"avg_ms": 1.097, "count": 224, "max_ms": 1.454, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.481, "count": 224, "max_ms": 68.112, "p95_ms": 63.272}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.354, "count": 224, "max_ms": 87.058, "p95_ms": 76.629}, "yolo11_640_fire_11class": {"avg_ms": 50.27, "count": 224, "max_ms": 67.595, "p95_ms": 62.252}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.941, "count": 224, "max_ms": 85.176, "p95_ms": 76.654}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.131, "count": 224, "max_ms": 149.408, "p95_ms": 140.233}, "read_interval": {"avg_ms": 133.759, "count": 224, "max_ms": 161.151, "p95_ms": 152.876}, "triton_wait": {"avg_ms": 60.035, "count": 224, "max_ms": 87.07, "p95_ms": 78.3}, "visualize": {"avg_ms": 4.717, "count": 224, "max_ms": 5.946, "p95_ms": 5.103}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 333, "tracking_summary": 15973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16257.375, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:35:00+09:00", "last_error_summary": "[h264 @ 0x557b313420c0] left block unavailable for requested intra mode [h264 @ 0x557b313420c0] error while decoding MB 0 35, bytestream 4703 [h264 @ 0x557b31f0c8c0] error while decoding MB 74 34, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 785, "h264_decode_error_count": 1924, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:35:00+09:00", "last_error_summary": "[h264 @ 0x557b313420c0] left block unavailable for requested intra mode [h264 @ 0x557b313420c0] error while decoding MB 0 35, bytestream 4703 [h264 @ 0x557b31f0c8c0] error while decoding MB 74 34, bytestream -6", "last_exit_at": "2026-07-06T05:35:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 200.338, "last_started_at": "2026-07-06T05:31:39+09:00", "pid": 146409, "process": {"exists": false, "pid": 146409}, "restart_count": 873, "restart_count_per_hour": 12, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 874}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.676, "sample_count": 2111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1916781, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:34:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T05:35:00+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-06T05:34:59+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-06T05:34:59.970715+09:00", "mtime_epoch": 1783283699.9707146, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168975}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:35:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9539.973, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 225, "read_frames": 224, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.508, "pushed": 7.508, "read": 7.475}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.387, "count": 224, "max_ms": 26.44, "p95_ms": 21.097}, "gstreamer_push": {"avg_ms": 1.098, "count": 225, "max_ms": 1.454, "p95_ms": 1.198}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.477, "count": 224, "max_ms": 68.112, "p95_ms": 63.272}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.328, "count": 224, "max_ms": 87.058, "p95_ms": 76.124}, "yolo11_640_fire_11class": {"avg_ms": 50.268, "count": 224, "max_ms": 67.595, "p95_ms": 62.252}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.932, "count": 224, "max_ms": 85.176, "p95_ms": 76.628}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.21, "count": 225, "max_ms": 149.408, "p95_ms": 140.233}, "read_interval": {"avg_ms": 133.759, "count": 224, "max_ms": 161.151, "p95_ms": 152.876}, "triton_wait": {"avg_ms": 60.01, "count": 224, "max_ms": 87.07, "p95_ms": 78.144}, "visualize": {"avg_ms": 4.717, "count": 224, "max_ms": 5.946, "p95_ms": 5.103}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 333, "tracking_summary": 15973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16257.445, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 785, "h264_decode_error_count": 1924, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:35:00+09:00", "last_error_summary": "[h264 @ 0x557b313420c0] left block unavailable for requested intra mode [h264 @ 0x557b313420c0] error while decoding MB 0 35, bytestream 4703 [h264 @ 0x557b31f0c8c0] error while decoding MB 74 34, bytestream -6", "last_exit_at": "2026-07-06T05:35:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 200.344, "last_started_at": "2026-07-06T05:31:39+09:00", "pid": 146409, "process": {"exists": false, "pid": 146409}, "restart_count": 873, "restart_count_per_hour": 12, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 874}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.676, "sample_count": 2111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1916781, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:34:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T05:35:00+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-06T05:34:59+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-06T05:34:59.970715+09:00", "mtime_epoch": 1783283699.9707146, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168975}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:35:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9539.973, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 225, "read_frames": 224, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.507, "pushed": 7.507, "read": 7.474}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.387, "count": 224, "max_ms": 26.44, "p95_ms": 21.097}, "gstreamer_push": {"avg_ms": 1.098, "count": 225, "max_ms": 1.454, "p95_ms": 1.198}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.477, "count": 224, "max_ms": 68.112, "p95_ms": 63.272}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.328, "count": 224, "max_ms": 87.058, "p95_ms": 76.124}, "yolo11_640_fire_11class": {"avg_ms": 50.268, "count": 224, "max_ms": 67.595, "p95_ms": 62.252}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.932, "count": 224, "max_ms": 85.176, "p95_ms": 76.628}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.21, "count": 225, "max_ms": 149.408, "p95_ms": 140.233}, "read_interval": {"avg_ms": 133.759, "count": 224, "max_ms": 161.151, "p95_ms": 152.876}, "triton_wait": {"avg_ms": 60.01, "count": 224, "max_ms": 87.07, "p95_ms": 78.144}, "visualize": {"avg_ms": 4.717, "count": 224, "max_ms": 5.946, "p95_ms": 5.103}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 333, "tracking_summary": 15973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16370.941, "limit_mb": 8796093022207.996}, "detail": {"pid": 146576, "restart_count": 874, "start_count": 875}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 785, "h264_decode_error_count": 1924, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:35:00+09:00", "last_error_summary": "[h264 @ 0x557b313420c0] left block unavailable for requested intra mode [h264 @ 0x557b313420c0] error while decoding MB 0 35, bytestream 4703 [h264 @ 0x557b31f0c8c0] error while decoding MB 74 34, bytestream -6", "last_exit_at": "2026-07-06T05:35:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:35:02+09:00", "pid": 146576, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.754, "name": "ffmpeg", "pid": 146576, "rss_mb": 144.754, "state": "S (sleeping)", "threads": 106}, "restart_count": 874, "restart_count_per_hour": 13, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 875}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.189, "sample_count": 2089, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1916781, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:34:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.422, "last_frame_pushed_at": "2026-07-06T05:35:00+09:00", "last_frame_read_age_sec": 4.563, "last_frame_read_at": "2026-07-06T05:34:59+09:00", "mjpeg": {"age_sec": 4.406, "exists": true, "mtime": "2026-07-06T05:35:00.118716+09:00", "mtime_epoch": 1783283700.1187158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170565}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:35:04+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9539.973, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 192, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.367, "pushed": 6.4, "read": 6.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.415, "count": 191, "max_ms": 22.271, "p95_ms": 21.097}, "gstreamer_push": {"avg_ms": 1.101, "count": 192, "max_ms": 1.454, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.504, "count": 191, "max_ms": 68.112, "p95_ms": 63.012}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.458, "count": 191, "max_ms": 87.058, "p95_ms": 76.124}, "yolo11_640_fire_11class": {"avg_ms": 50.581, "count": 191, "max_ms": 67.595, "p95_ms": 62.499}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.108, "count": 191, "max_ms": 85.176, "p95_ms": 76.628}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.273, "count": 191, "max_ms": 149.408, "p95_ms": 140.0}, "read_interval": {"avg_ms": 133.763, "count": 191, "max_ms": 161.151, "p95_ms": 152.876}, "triton_wait": {"avg_ms": 60.137, "count": 191, "max_ms": 87.07, "p95_ms": 78.144}, "visualize": {"avg_ms": 4.732, "count": 191, "max_ms": 5.946, "p95_ms": 5.127}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 334, "tracking_summary": 15982}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16264.758, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5593a5745b80] error while decoding MB 3 39, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 786, "h264_decode_error_count": 1925, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:37:43+09:00", "last_error_summary": "[h264 @ 0x5593a5745b80] error while decoding MB 3 39, bytestream -7", "last_exit_at": "2026-07-06T05:35:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 161.047, "last_started_at": "2026-07-06T05:35:02+09:00", "pid": 146576, "process": {"exists": false, "pid": 146576}, "restart_count": 874, "restart_count_per_hour": 13, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 875}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.254, "sample_count": 2103, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1917858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:37:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T05:37:43+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T05:37:43+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T05:37:43.524110+09:00", "mtime_epoch": 1783283863.52411, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173554}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:37:43+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9540.488, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.54, "pushed": 7.54, "read": 7.54}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.897, "count": 225, "max_ms": 32.18, "p95_ms": 20.843}, "gstreamer_push": {"avg_ms": 1.098, "count": 226, "max_ms": 1.437, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.174, "count": 225, "max_ms": 70.922, "p95_ms": 62.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.343, "count": 225, "max_ms": 85.161, "p95_ms": 77.064}, "yolo11_640_fire_11class": {"avg_ms": 51.243, "count": 225, "max_ms": 71.369, "p95_ms": 66.201}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.638, "count": 225, "max_ms": 80.636, "p95_ms": 77.221}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.737, "count": 226, "max_ms": 150.695, "p95_ms": 140.171}, "read_interval": {"avg_ms": 132.92, "count": 226, "max_ms": 164.576, "p95_ms": 152.893}, "triton_wait": {"avg_ms": 59.929, "count": 225, "max_ms": 85.172, "p95_ms": 78.171}, "visualize": {"avg_ms": 4.652, "count": 225, "max_ms": 5.815, "p95_ms": 5.062}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 334, "tracking_summary": 15982}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16264.43, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:37:43+09:00", "last_error_summary": "[h264 @ 0x5593a5745b80] error while decoding MB 3 39, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 786, "h264_decode_error_count": 1925, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:37:43+09:00", "last_error_summary": "[h264 @ 0x5593a5745b80] error while decoding MB 3 39, bytestream -7", "last_exit_at": "2026-07-06T05:37:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 161.13, "last_started_at": "2026-07-06T05:35:02+09:00", "pid": 146576, "process": {"exists": false, "pid": 146576}, "restart_count": 874, "restart_count_per_hour": 13, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 875}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.254, "sample_count": 2103, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1917859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:37:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T05:37:43+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T05:37:43+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T05:37:43.524110+09:00", "mtime_epoch": 1783283863.52411, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173554}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:37:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9540.488, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.557, "pushed": 7.557, "read": 7.524}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.913, "count": 226, "max_ms": 32.18, "p95_ms": 21.297}, "gstreamer_push": {"avg_ms": 1.098, "count": 226, "max_ms": 1.437, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.154, "count": 226, "max_ms": 70.922, "p95_ms": 62.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.339, "count": 226, "max_ms": 85.161, "p95_ms": 77.064}, "yolo11_640_fire_11class": {"avg_ms": 51.214, "count": 226, "max_ms": 71.369, "p95_ms": 66.201}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.624, "count": 226, "max_ms": 80.636, "p95_ms": 77.221}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.785, "count": 226, "max_ms": 150.695, "p95_ms": 140.171}, "read_interval": {"avg_ms": 132.946, "count": 225, "max_ms": 164.576, "p95_ms": 152.893}, "triton_wait": {"avg_ms": 59.91, "count": 226, "max_ms": 85.172, "p95_ms": 78.171}, "visualize": {"avg_ms": 4.655, "count": 226, "max_ms": 5.815, "p95_ms": 5.123}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 334, "tracking_summary": 15982}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16264.426, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 786, "h264_decode_error_count": 1925, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:37:43+09:00", "last_error_summary": "[h264 @ 0x5593a5745b80] error while decoding MB 3 39, bytestream -7", "last_exit_at": "2026-07-06T05:37:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 161.136, "last_started_at": "2026-07-06T05:35:02+09:00", "pid": 146576, "process": {"exists": false, "pid": 146576}, "restart_count": 874, "restart_count_per_hour": 13, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 875}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.254, "sample_count": 2103, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1917859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:37:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T05:37:43+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T05:37:43+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-06T05:37:43.524110+09:00", "mtime_epoch": 1783283863.52411, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173554}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:37:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9540.488, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.556, "pushed": 7.556, "read": 7.522}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.913, "count": 226, "max_ms": 32.18, "p95_ms": 21.297}, "gstreamer_push": {"avg_ms": 1.098, "count": 226, "max_ms": 1.437, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.154, "count": 226, "max_ms": 70.922, "p95_ms": 62.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.339, "count": 226, "max_ms": 85.161, "p95_ms": 77.064}, "yolo11_640_fire_11class": {"avg_ms": 51.214, "count": 226, "max_ms": 71.369, "p95_ms": 66.201}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.624, "count": 226, "max_ms": 80.636, "p95_ms": 77.221}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.785, "count": 226, "max_ms": 150.695, "p95_ms": 140.171}, "read_interval": {"avg_ms": 132.946, "count": 225, "max_ms": 164.576, "p95_ms": 152.893}, "triton_wait": {"avg_ms": 59.91, "count": 226, "max_ms": 85.172, "p95_ms": 78.171}, "visualize": {"avg_ms": 4.655, "count": 226, "max_ms": 5.815, "p95_ms": 5.123}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 334, "tracking_summary": 15982}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16377.512, "limit_mb": 8796093022207.996}, "detail": {"pid": 146743, "restart_count": 875, "start_count": 876}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 786, "h264_decode_error_count": 1925, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:37:43+09:00", "last_error_summary": "[h264 @ 0x5593a5745b80] error while decoding MB 3 39, bytestream -7", "last_exit_at": "2026-07-06T05:37:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:37:46+09:00", "pid": 146743, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.781, "name": "ffmpeg", "pid": 146743, "rss_mb": 144.781, "state": "S (sleeping)", "threads": 106}, "restart_count": 875, "restart_count_per_hour": 14, "rtsp_timeout_count": 496, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 876}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.862, "sample_count": 2067, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1917859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:37:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.744, "last_frame_pushed_at": "2026-07-06T05:37:43+09:00", "last_frame_read_age_sec": 4.872, "last_frame_read_at": "2026-07-06T05:37:43+09:00", "mjpeg": {"age_sec": 4.729, "exists": true, "mtime": "2026-07-06T05:37:43.664111+09:00", "mtime_epoch": 1783283863.6641114, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173763}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:37:48+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9540.488, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.385, "pushed": 6.385, "read": 6.352}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.813, "count": 191, "max_ms": 31.999, "p95_ms": 21.862}, "gstreamer_push": {"avg_ms": 1.094, "count": 191, "max_ms": 1.437, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.02, "count": 191, "max_ms": 70.922, "p95_ms": 63.317}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.806, "count": 191, "max_ms": 85.161, "p95_ms": 76.226}, "yolo11_640_fire_11class": {"avg_ms": 51.179, "count": 191, "max_ms": 71.369, "p95_ms": 66.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.471, "count": 191, "max_ms": 80.636, "p95_ms": 77.687}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.253, "count": 191, "max_ms": 148.733, "p95_ms": 140.171}, "read_interval": {"avg_ms": 132.353, "count": 190, "max_ms": 162.256, "p95_ms": 152.893}, "triton_wait": {"avg_ms": 59.726, "count": 191, "max_ms": 85.172, "p95_ms": 78.171}, "visualize": {"avg_ms": 4.631, "count": 191, "max_ms": 5.676, "p95_ms": 4.899}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 334, "tracking_summary": 15993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16266.723, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 787, "h264_decode_error_count": 1931, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:41:09+09:00", "last_error_summary": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "last_exit_at": "2026-07-06T05:37:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.861, "last_started_at": "2026-07-06T05:37:46+09:00", "pid": 146743, "process": {"exists": false, "pid": 146743}, "restart_count": 875, "restart_count_per_hour": 14, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 876}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.657, "sample_count": 2014, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1919234, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:41:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T05:41:09+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T05:41:09+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T05:41:09.210166+09:00", "mtime_epoch": 1783284069.2101662, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:41:09+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9541.262, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.304, "pushed": 3.304, "read": 3.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.004, "count": 99, "max_ms": 23.789, "p95_ms": 20.327}, "gstreamer_push": {"avg_ms": 1.122, "count": 99, "max_ms": 1.39, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.862, "count": 99, "max_ms": 63.703, "p95_ms": 62.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.662, "count": 99, "max_ms": 75.976, "p95_ms": 73.227}, "yolo11_640_fire_11class": {"avg_ms": 51.641, "count": 99, "max_ms": 67.683, "p95_ms": 66.314}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.396, "count": 99, "max_ms": 80.448, "p95_ms": 78.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.432, "count": 99, "max_ms": 141.117, "p95_ms": 139.959}, "read_interval": {"avg_ms": 303.246, "count": 99, "max_ms": 16872.063, "p95_ms": 152.798}, "triton_wait": {"avg_ms": 60.583, "count": 99, "max_ms": 80.459, "p95_ms": 78.462}, "visualize": {"avg_ms": 4.727, "count": 99, "max_ms": 5.901, "p95_ms": 4.926}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 334, "tracking_summary": 15993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16266.422, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:41:09+09:00", "last_error_summary": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 787, "h264_decode_error_count": 1931, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:41:09+09:00", "last_error_summary": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "last_exit_at": "2026-07-06T05:41:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.941, "last_started_at": "2026-07-06T05:37:46+09:00", "pid": 146743, "process": {"exists": false, "pid": 146743}, "restart_count": 875, "restart_count_per_hour": 14, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 876}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.657, "sample_count": 2015, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1919235, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:41:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T05:41:09+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-06T05:41:09+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-06T05:41:09.210166+09:00", "mtime_epoch": 1783284069.2101662, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:41:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9541.262, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.311, "pushed": 3.311, "read": 3.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.021, "count": 99, "max_ms": 23.789, "p95_ms": 20.327}, "gstreamer_push": {"avg_ms": 1.124, "count": 99, "max_ms": 1.39, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.876, "count": 99, "max_ms": 63.703, "p95_ms": 62.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.671, "count": 99, "max_ms": 75.976, "p95_ms": 73.227}, "yolo11_640_fire_11class": {"avg_ms": 51.659, "count": 99, "max_ms": 67.683, "p95_ms": 66.314}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.435, "count": 99, "max_ms": 80.448, "p95_ms": 78.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.552, "count": 99, "max_ms": 141.117, "p95_ms": 139.959}, "read_interval": {"avg_ms": 305.057, "count": 98, "max_ms": 16872.063, "p95_ms": 152.798}, "triton_wait": {"avg_ms": 60.622, "count": 99, "max_ms": 80.459, "p95_ms": 78.462}, "visualize": {"avg_ms": 4.736, "count": 99, "max_ms": 5.901, "p95_ms": 4.936}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 334, "tracking_summary": 15993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16266.41, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 787, "h264_decode_error_count": 1931, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:41:09+09:00", "last_error_summary": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "last_exit_at": "2026-07-06T05:41:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.947, "last_started_at": "2026-07-06T05:37:46+09:00", "pid": 146743, "process": {"exists": false, "pid": 146743}, "restart_count": 875, "restart_count_per_hour": 14, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 876}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.657, "sample_count": 2015, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1919235, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:41:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T05:41:09+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-06T05:41:09+09:00", "mjpeg": {"age_sec": 0.129, "exists": true, "mtime": "2026-07-06T05:41:09.210166+09:00", "mtime_epoch": 1783284069.2101662, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:41:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9541.262, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.31, "pushed": 3.31, "read": 3.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.021, "count": 99, "max_ms": 23.789, "p95_ms": 20.327}, "gstreamer_push": {"avg_ms": 1.124, "count": 99, "max_ms": 1.39, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.876, "count": 99, "max_ms": 63.703, "p95_ms": 62.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.671, "count": 99, "max_ms": 75.976, "p95_ms": 73.227}, "yolo11_640_fire_11class": {"avg_ms": 51.659, "count": 99, "max_ms": 67.683, "p95_ms": 66.314}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.435, "count": 99, "max_ms": 80.448, "p95_ms": 78.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.552, "count": 99, "max_ms": 141.117, "p95_ms": 139.959}, "read_interval": {"avg_ms": 305.057, "count": 98, "max_ms": 16872.063, "p95_ms": 152.798}, "triton_wait": {"avg_ms": 60.622, "count": 99, "max_ms": 80.459, "p95_ms": 78.462}, "visualize": {"avg_ms": 4.736, "count": 99, "max_ms": 5.901, "p95_ms": 4.936}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 334, "tracking_summary": 15993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16381.977, "limit_mb": 8796093022207.996}, "detail": {"pid": 146850, "restart_count": 876, "start_count": 877}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 787, "h264_decode_error_count": 1931, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:41:09+09:00", "last_error_summary": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "last_exit_at": "2026-07-06T05:41:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:41:11+09:00", "pid": 146850, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.707, "name": "ffmpeg", "pid": 146850, "rss_mb": 146.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 876, "restart_count_per_hour": 15, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 877}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.818, "sample_count": 1989, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1919235, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:41:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.187, "last_frame_pushed_at": "2026-07-06T05:41:09+09:00", "last_frame_read_age_sec": 4.314, "last_frame_read_at": "2026-07-06T05:41:09+09:00", "mjpeg": {"age_sec": 4.17, "exists": true, "mtime": "2026-07-06T05:41:09.346168+09:00", "mtime_epoch": 1783284069.3461676, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178278}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:41:13+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9541.262, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.271, "pushed": 2.271, "read": 2.238}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.121, "count": 68, "max_ms": 23.789, "p95_ms": 20.327}, "gstreamer_push": {"avg_ms": 1.124, "count": 68, "max_ms": 1.39, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.612, "count": 68, "max_ms": 63.703, "p95_ms": 61.946}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.778, "count": 68, "max_ms": 75.976, "p95_ms": 73.397}, "yolo11_640_fire_11class": {"avg_ms": 51.382, "count": 68, "max_ms": 67.458, "p95_ms": 65.989}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.279, "count": 68, "max_ms": 78.906, "p95_ms": 78.04}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.515, "count": 68, "max_ms": 140.559, "p95_ms": 139.894}, "read_interval": {"avg_ms": 384.073, "count": 67, "max_ms": 16872.063, "p95_ms": 152.798}, "triton_wait": {"avg_ms": 60.485, "count": 68, "max_ms": 78.918, "p95_ms": 78.051}, "visualize": {"avg_ms": 4.749, "count": 68, "max_ms": 5.901, "p95_ms": 4.936}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 335, "tracking_summary": 16005}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16272.637, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:41:09+09:00", "last_error_summary": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 787, "h264_decode_error_count": 1931, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:41:09+09:00", "last_error_summary": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "last_exit_at": "2026-07-06T05:44:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.143, "last_started_at": "2026-07-06T05:41:11+09:00", "pid": 146850, "process": {"exists": false, "pid": 146850}, "restart_count": 876, "restart_count_per_hour": 15, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 877}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.956, "sample_count": 2049, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1920630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:44:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T05:44:24+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-06T05:44:24+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T05:44:24.536334+09:00", "mtime_epoch": 1783284264.536334, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:44:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9541.777, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.452, "pushed": 7.452, "read": 7.418}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.588, "count": 223, "max_ms": 26.349, "p95_ms": 20.03}, "gstreamer_push": {"avg_ms": 1.117, "count": 223, "max_ms": 1.584, "p95_ms": 1.182}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.107, "count": 223, "max_ms": 67.397, "p95_ms": 62.861}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.181, "count": 223, "max_ms": 79.485, "p95_ms": 75.176}, "yolo11_640_fire_11class": {"avg_ms": 51.766, "count": 223, "max_ms": 71.857, "p95_ms": 65.727}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.232, "count": 223, "max_ms": 83.499, "p95_ms": 77.465}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.143, "count": 223, "max_ms": 145.374, "p95_ms": 139.218}, "read_interval": {"avg_ms": 134.766, "count": 222, "max_ms": 162.152, "p95_ms": 151.723}, "triton_wait": {"avg_ms": 61.372, "count": 223, "max_ms": 83.51, "p95_ms": 78.137}, "visualize": {"avg_ms": 4.801, "count": 223, "max_ms": 6.856, "p95_ms": 5.673}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 335, "tracking_summary": 16005}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16272.801, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 787, "h264_decode_error_count": 1931, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:41:09+09:00", "last_error_summary": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "last_exit_at": "2026-07-06T05:44:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.148, "last_started_at": "2026-07-06T05:41:11+09:00", "pid": 146850, "process": {"exists": false, "pid": 146850}, "restart_count": 876, "restart_count_per_hour": 15, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 877}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.956, "sample_count": 2049, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1920630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:44:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T05:44:24+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T05:44:24+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-06T05:44:24.536334+09:00", "mtime_epoch": 1783284264.536334, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:44:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9541.777, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.45, "pushed": 7.45, "read": 7.417}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.588, "count": 223, "max_ms": 26.349, "p95_ms": 20.03}, "gstreamer_push": {"avg_ms": 1.117, "count": 223, "max_ms": 1.584, "p95_ms": 1.182}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.107, "count": 223, "max_ms": 67.397, "p95_ms": 62.861}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.181, "count": 223, "max_ms": 79.485, "p95_ms": 75.176}, "yolo11_640_fire_11class": {"avg_ms": 51.766, "count": 223, "max_ms": 71.857, "p95_ms": 65.727}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.232, "count": 223, "max_ms": 83.499, "p95_ms": 77.465}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.143, "count": 223, "max_ms": 145.374, "p95_ms": 139.218}, "read_interval": {"avg_ms": 134.766, "count": 222, "max_ms": 162.152, "p95_ms": 151.723}, "triton_wait": {"avg_ms": 61.372, "count": 223, "max_ms": 83.51, "p95_ms": 78.137}, "visualize": {"avg_ms": 4.801, "count": 223, "max_ms": 6.856, "p95_ms": 5.673}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 335, "tracking_summary": 16005}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16381.293, "limit_mb": 8796093022207.996}, "detail": {"pid": 147017, "restart_count": 877, "start_count": 878}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 787, "h264_decode_error_count": 1931, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:41:09+09:00", "last_error_summary": "[h264 @ 0x563437598ac0] cabac decode of qscale diff failed at 5 42 [h264 @ 0x563437598ac0] error while decoding MB 5 42, bytestream 4665 [h264 @ 0x5634375db380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5634375db380] error while decoding MB 0 30, bytestream 3603 [h264 @ 0x563437e5ff80] cabac decode of qscale diff failed at 26 33 [h264 @ 0x563437e5ff80] error while decoding MB 26 33, bytestream 10029 [h264 @ 0x563437ffdc80] error while decoding MB 20 44, bytestream -13 [in#0", "last_exit_at": "2026-07-06T05:44:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:44:27+09:00", "pid": 147017, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.98, "name": "ffmpeg", "pid": 147017, "rss_mb": 138.98, "state": "S (sleeping)", "threads": 106}, "restart_count": 877, "restart_count_per_hour": 16, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 878}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.955, "sample_count": 2016, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1920630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:44:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.509, "last_frame_pushed_at": "2026-07-06T05:44:24+09:00", "last_frame_read_age_sec": 4.627, "last_frame_read_at": "2026-07-06T05:44:24+09:00", "mjpeg": {"age_sec": 4.493, "exists": true, "mtime": "2026-07-06T05:44:24.668335+09:00", "mtime_epoch": 1783284264.6683354, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168323}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:44:29+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9541.777, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.335, "pushed": 6.335, "read": 6.302}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.489, "count": 189, "max_ms": 22.585, "p95_ms": 19.805}, "gstreamer_push": {"avg_ms": 1.113, "count": 190, "max_ms": 1.584, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.36, "count": 189, "max_ms": 67.397, "p95_ms": 63.131}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.493, "count": 189, "max_ms": 79.485, "p95_ms": 76.413}, "yolo11_640_fire_11class": {"avg_ms": 52.019, "count": 189, "max_ms": 71.857, "p95_ms": 65.807}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.626, "count": 189, "max_ms": 83.499, "p95_ms": 77.876}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.297, "count": 190, "max_ms": 145.374, "p95_ms": 139.456}, "read_interval": {"avg_ms": 134.835, "count": 189, "max_ms": 162.152, "p95_ms": 151.937}, "triton_wait": {"avg_ms": 61.747, "count": 189, "max_ms": 83.51, "p95_ms": 78.471}, "visualize": {"avg_ms": 4.736, "count": 189, "max_ms": 6.07, "p95_ms": 5.056}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16013}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16279.809, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a856966c40] error while decoding MB 43 25, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 788, "h264_decode_error_count": 1932, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:47:01+09:00", "last_error_summary": "[h264 @ 0x55a856966c40] error while decoding MB 43 25, bytestream -12", "last_exit_at": "2026-07-06T05:44:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 154.075, "last_started_at": "2026-07-06T05:44:27+09:00", "pid": 147017, "process": {"exists": false, "pid": 147017}, "restart_count": 877, "restart_count_per_hour": 15, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 878}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.209, "sample_count": 2039, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1921629, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:47:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T05:47:01+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T05:47:01+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T05:47:01.194178+09:00", "mtime_epoch": 1783284421.1941779, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159852}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:47:01+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9542.293, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.869, "pushed": 6.869, "read": 6.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.672, "count": 205, "max_ms": 27.146, "p95_ms": 22.714}, "gstreamer_push": {"avg_ms": 1.074, "count": 206, "max_ms": 1.436, "p95_ms": 1.167}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.199, "count": 205, "max_ms": 64.536, "p95_ms": 61.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.978, "count": 205, "max_ms": 83.074, "p95_ms": 78.328}, "yolo11_640_fire_11class": {"avg_ms": 52.584, "count": 205, "max_ms": 72.142, "p95_ms": 66.772}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.623, "count": 205, "max_ms": 82.151, "p95_ms": 79.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.089, "count": 206, "max_ms": 242.63, "p95_ms": 183.945}, "read_interval": {"avg_ms": 146.211, "count": 206, "max_ms": 252.863, "p95_ms": 194.095}, "triton_wait": {"avg_ms": 63.726, "count": 205, "max_ms": 83.086, "p95_ms": 80.402}, "visualize": {"avg_ms": 5.472, "count": 205, "max_ms": 12.415, "p95_ms": 10.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16013}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16279.215, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:47:01+09:00", "last_error_summary": "[h264 @ 0x55a856966c40] error while decoding MB 43 25, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 788, "h264_decode_error_count": 1932, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:47:01+09:00", "last_error_summary": "[h264 @ 0x55a856966c40] error while decoding MB 43 25, bytestream -12", "last_exit_at": "2026-07-06T05:47:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 154.164, "last_started_at": "2026-07-06T05:44:27+09:00", "pid": 147017, "process": {"exists": false, "pid": 147017}, "restart_count": 877, "restart_count_per_hour": 15, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 878}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.209, "sample_count": 2040, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1921630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:47:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T05:47:01+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T05:47:01+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T05:47:01.194178+09:00", "mtime_epoch": 1783284421.1941779, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159852}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:47:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9542.293, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.894, "pushed": 6.894, "read": 6.86}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.662, "count": 206, "max_ms": 27.146, "p95_ms": 22.714}, "gstreamer_push": {"avg_ms": 1.075, "count": 206, "max_ms": 1.436, "p95_ms": 1.167}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.217, "count": 206, "max_ms": 64.536, "p95_ms": 61.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.044, "count": 206, "max_ms": 83.074, "p95_ms": 78.328}, "yolo11_640_fire_11class": {"avg_ms": 52.605, "count": 206, "max_ms": 72.142, "p95_ms": 66.772}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.673, "count": 206, "max_ms": 82.151, "p95_ms": 79.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.859, "count": 206, "max_ms": 242.63, "p95_ms": 183.945}, "read_interval": {"avg_ms": 145.993, "count": 205, "max_ms": 252.863, "p95_ms": 194.095}, "triton_wait": {"avg_ms": 63.773, "count": 206, "max_ms": 83.086, "p95_ms": 80.402}, "visualize": {"avg_ms": 5.467, "count": 206, "max_ms": 12.415, "p95_ms": 10.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16013}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16279.117, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 788, "h264_decode_error_count": 1932, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:47:01+09:00", "last_error_summary": "[h264 @ 0x55a856966c40] error while decoding MB 43 25, bytestream -12", "last_exit_at": "2026-07-06T05:47:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 154.17, "last_started_at": "2026-07-06T05:44:27+09:00", "pid": 147017, "process": {"exists": false, "pid": 147017}, "restart_count": 877, "restart_count_per_hour": 15, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 878}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.209, "sample_count": 2040, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1921630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:47:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T05:47:01+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-06T05:47:01+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-06T05:47:01.194178+09:00", "mtime_epoch": 1783284421.1941779, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159852}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:47:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9542.293, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.893, "pushed": 6.893, "read": 6.859}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.662, "count": 206, "max_ms": 27.146, "p95_ms": 22.714}, "gstreamer_push": {"avg_ms": 1.075, "count": 206, "max_ms": 1.436, "p95_ms": 1.167}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.217, "count": 206, "max_ms": 64.536, "p95_ms": 61.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.044, "count": 206, "max_ms": 83.074, "p95_ms": 78.328}, "yolo11_640_fire_11class": {"avg_ms": 52.605, "count": 206, "max_ms": 72.142, "p95_ms": 66.772}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.673, "count": 206, "max_ms": 82.151, "p95_ms": 79.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.859, "count": 206, "max_ms": 242.63, "p95_ms": 183.945}, "read_interval": {"avg_ms": 145.993, "count": 205, "max_ms": 252.863, "p95_ms": 194.095}, "triton_wait": {"avg_ms": 63.773, "count": 206, "max_ms": 83.086, "p95_ms": 80.402}, "visualize": {"avg_ms": 5.467, "count": 206, "max_ms": 12.415, "p95_ms": 10.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16013}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16386.824, "limit_mb": 8796093022207.996}, "detail": {"pid": 147184, "restart_count": 878, "start_count": 879}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 788, "h264_decode_error_count": 1932, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:47:01+09:00", "last_error_summary": "[h264 @ 0x55a856966c40] error while decoding MB 43 25, bytestream -12", "last_exit_at": "2026-07-06T05:47:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:47:04+09:00", "pid": 147184, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.961, "name": "ffmpeg", "pid": 147184, "rss_mb": 138.961, "state": "S (sleeping)", "threads": 106}, "restart_count": 878, "restart_count_per_hour": 16, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 879}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.379, "sample_count": 2005, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1921630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:47:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.747, "last_frame_pushed_at": "2026-07-06T05:47:01+09:00", "last_frame_read_age_sec": 4.882, "last_frame_read_at": "2026-07-06T05:47:01+09:00", "mjpeg": {"age_sec": 4.729, "exists": true, "mtime": "2026-07-06T05:47:01.338180+09:00", "mtime_epoch": 1783284421.3381796, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160023}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:47:06+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9542.293, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.044, "pushed": 6.044, "read": 6.011}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.341, "count": 181, "max_ms": 27.146, "p95_ms": 21.516}, "gstreamer_push": {"avg_ms": 1.1, "count": 181, "max_ms": 1.436, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.655, "count": 181, "max_ms": 64.536, "p95_ms": 61.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.398, "count": 181, "max_ms": 83.074, "p95_ms": 78.201}, "yolo11_640_fire_11class": {"avg_ms": 53.384, "count": 181, "max_ms": 72.142, "p95_ms": 66.772}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.206, "count": 181, "max_ms": 82.151, "p95_ms": 79.815}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.609, "count": 181, "max_ms": 145.391, "p95_ms": 142.694}, "read_interval": {"avg_ms": 139.932, "count": 180, "max_ms": 157.455, "p95_ms": 155.475}, "triton_wait": {"avg_ms": 64.217, "count": 181, "max_ms": 83.086, "p95_ms": 79.993}, "visualize": {"avg_ms": 4.796, "count": 181, "max_ms": 6.854, "p95_ms": 5.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16025}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16280.785, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561c3d13b940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c3d13b940] error while decoding MB 0 23, bytestream 3210 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 48 44 [h264 @ 0x561c3d059f80] error while decoding MB 48 44, bytestream 876 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 27 42 [h264 @ 0x561c3d059f80] error while decoding MB 27 42, bytestream 1567", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 789, "h264_decode_error_count": 1937, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:50:27+09:00", "last_error_summary": "[h264 @ 0x561c3d13b940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c3d13b940] error while decoding MB 0 23, bytestream 3210 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 48 44 [h264 @ 0x561c3d059f80] error while decoding MB 48 44, bytestream 876 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 27 42 [h264 @ 0x561c3d059f80] error while decoding MB 27 42, bytestream 1567", "last_exit_at": "2026-07-06T05:47:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 203.801, "last_started_at": "2026-07-06T05:47:04+09:00", "pid": 147184, "process": {"exists": false, "pid": 147184}, "restart_count": 878, "restart_count_per_hour": 15, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 879}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.438, "sample_count": 2058, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1923088, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:50:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-06T05:50:27+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-06T05:50:27+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T05:50:27.812709+09:00", "mtime_epoch": 1783284627.812709, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148459}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:50:27+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9543.066, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.455, "pushed": 7.455, "read": 7.455}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.929, "count": 223, "max_ms": 29.123, "p95_ms": 20.68}, "gstreamer_push": {"avg_ms": 1.103, "count": 223, "max_ms": 1.392, "p95_ms": 1.201}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.922, "count": 223, "max_ms": 66.186, "p95_ms": 61.638}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.167, "count": 223, "max_ms": 82.864, "p95_ms": 78.199}, "yolo11_640_fire_11class": {"avg_ms": 51.983, "count": 223, "max_ms": 67.913, "p95_ms": 64.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.8, "count": 223, "max_ms": 79.301, "p95_ms": 77.191}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.596, "count": 223, "max_ms": 145.339, "p95_ms": 139.845}, "read_interval": {"avg_ms": 134.229, "count": 223, "max_ms": 157.561, "p95_ms": 152.179}, "triton_wait": {"avg_ms": 62.896, "count": 223, "max_ms": 82.877, "p95_ms": 78.512}, "visualize": {"avg_ms": 4.581, "count": 223, "max_ms": 5.356, "p95_ms": 4.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16025}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16280.535, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:50:27+09:00", "last_error_summary": "[h264 @ 0x561c3d13b940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c3d13b940] error while decoding MB 0 23, bytestream 3210 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 48 44 [h264 @ 0x561c3d059f80] error while decoding MB 48 44, bytestream 876 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 27 42 [h264 @ 0x561c3d059f80] error while decoding MB 27 42, bytestream 1567", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 789, "h264_decode_error_count": 1937, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:50:27+09:00", "last_error_summary": "[h264 @ 0x561c3d13b940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c3d13b940] error while decoding MB 0 23, bytestream 3210 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 48 44 [h264 @ 0x561c3d059f80] error while decoding MB 48 44, bytestream 876 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 27 42 [h264 @ 0x561c3d059f80] error while decoding MB 27 42, bytestream 1567", "last_exit_at": "2026-07-06T05:50:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 203.889, "last_started_at": "2026-07-06T05:47:04+09:00", "pid": 147184, "process": {"exists": false, "pid": 147184}, "restart_count": 878, "restart_count_per_hour": 15, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 879}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.438, "sample_count": 2058, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1923089, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:50:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T05:50:27+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-06T05:50:27+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T05:50:27.812709+09:00", "mtime_epoch": 1783284627.812709, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148459}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:50:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9543.066, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 224, "pushed_frames": 224, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.471, "pushed": 7.471, "read": 7.438}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.987, "count": 224, "max_ms": 32.059, "p95_ms": 20.723}, "gstreamer_push": {"avg_ms": 1.103, "count": 224, "max_ms": 1.392, "p95_ms": 1.201}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.897, "count": 224, "max_ms": 66.186, "p95_ms": 61.638}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.145, "count": 224, "max_ms": 82.864, "p95_ms": 78.199}, "yolo11_640_fire_11class": {"avg_ms": 51.955, "count": 224, "max_ms": 67.913, "p95_ms": 64.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.768, "count": 224, "max_ms": 79.301, "p95_ms": 77.191}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.666, "count": 224, "max_ms": 145.339, "p95_ms": 139.845}, "read_interval": {"avg_ms": 134.229, "count": 223, "max_ms": 157.561, "p95_ms": 152.179}, "triton_wait": {"avg_ms": 62.861, "count": 224, "max_ms": 82.877, "p95_ms": 78.512}, "visualize": {"avg_ms": 4.594, "count": 224, "max_ms": 7.464, "p95_ms": 4.849}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16025}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16280.613, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 789, "h264_decode_error_count": 1937, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:50:27+09:00", "last_error_summary": "[h264 @ 0x561c3d13b940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c3d13b940] error while decoding MB 0 23, bytestream 3210 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 48 44 [h264 @ 0x561c3d059f80] error while decoding MB 48 44, bytestream 876 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 27 42 [h264 @ 0x561c3d059f80] error while decoding MB 27 42, bytestream 1567", "last_exit_at": "2026-07-06T05:50:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 203.894, "last_started_at": "2026-07-06T05:47:04+09:00", "pid": 147184, "process": {"exists": false, "pid": 147184}, "restart_count": 878, "restart_count_per_hour": 15, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 879}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.438, "sample_count": 2058, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1923089, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:50:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T05:50:27+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T05:50:27+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-06T05:50:27.812709+09:00", "mtime_epoch": 1783284627.812709, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148459}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:50:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9543.066, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 224, "pushed_frames": 224, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.47, "pushed": 7.47, "read": 7.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.987, "count": 224, "max_ms": 32.059, "p95_ms": 20.723}, "gstreamer_push": {"avg_ms": 1.103, "count": 224, "max_ms": 1.392, "p95_ms": 1.201}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.897, "count": 224, "max_ms": 66.186, "p95_ms": 61.638}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.145, "count": 224, "max_ms": 82.864, "p95_ms": 78.199}, "yolo11_640_fire_11class": {"avg_ms": 51.955, "count": 224, "max_ms": 67.913, "p95_ms": 64.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.768, "count": 224, "max_ms": 79.301, "p95_ms": 77.191}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.666, "count": 224, "max_ms": 145.339, "p95_ms": 139.845}, "read_interval": {"avg_ms": 134.229, "count": 223, "max_ms": 157.561, "p95_ms": 152.179}, "triton_wait": {"avg_ms": 62.861, "count": 224, "max_ms": 82.877, "p95_ms": 78.512}, "visualize": {"avg_ms": 4.594, "count": 224, "max_ms": 7.464, "p95_ms": 4.849}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16025}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16388.961, "limit_mb": 8796093022207.996}, "detail": {"pid": 147291, "restart_count": 879, "start_count": 880}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 789, "h264_decode_error_count": 1937, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:50:27+09:00", "last_error_summary": "[h264 @ 0x561c3d13b940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c3d13b940] error while decoding MB 0 23, bytestream 3210 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 48 44 [h264 @ 0x561c3d059f80] error while decoding MB 48 44, bytestream 876 [h264 @ 0x561c3d059f80] cabac decode of qscale diff failed at 27 42 [h264 @ 0x561c3d059f80] error while decoding MB 27 42, bytestream 1567", "last_exit_at": "2026-07-06T05:50:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:50:30+09:00", "pid": 147291, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.914, "name": "ffmpeg", "pid": 147291, "rss_mb": 138.914, "state": "S (sleeping)", "threads": 106}, "restart_count": 879, "restart_count_per_hour": 16, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 880}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.438, "sample_count": 2029, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1923089, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:50:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.456, "last_frame_pushed_at": "2026-07-06T05:50:27+09:00", "last_frame_read_age_sec": 4.595, "last_frame_read_at": "2026-07-06T05:50:27+09:00", "mjpeg": {"age_sec": 4.44, "exists": true, "mtime": "2026-07-06T05:50:27.964711+09:00", "mtime_epoch": 1783284627.964711, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:50:32+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9543.066, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 186, "pushed_frames": 186, "read_frames": 185, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.217, "pushed": 6.217, "read": 6.183}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.071, "count": 186, "max_ms": 32.059, "p95_ms": 20.68}, "gstreamer_push": {"avg_ms": 1.093, "count": 186, "max_ms": 1.262, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.4, "count": 186, "max_ms": 66.186, "p95_ms": 61.738}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.939, "count": 186, "max_ms": 82.864, "p95_ms": 78.486}, "yolo11_640_fire_11class": {"avg_ms": 52.883, "count": 186, "max_ms": 67.913, "p95_ms": 64.921}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.495, "count": 186, "max_ms": 79.301, "p95_ms": 77.43}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.097, "count": 186, "max_ms": 145.339, "p95_ms": 140.096}, "read_interval": {"avg_ms": 137.494, "count": 185, "max_ms": 157.561, "p95_ms": 152.552}, "triton_wait": {"avg_ms": 63.756, "count": 186, "max_ms": 82.877, "p95_ms": 79.148}, "visualize": {"avg_ms": 4.684, "count": 186, "max_ms": 7.464, "p95_ms": 4.886}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16036}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16282.953, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 790, "h264_decode_error_count": 1938, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:53:29+09:00", "last_error_summary": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "last_exit_at": "2026-07-06T05:50:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.699, "last_started_at": "2026-07-06T05:50:30+09:00", "pid": 147291, "process": {"exists": false, "pid": 147291}, "restart_count": 879, "restart_count_per_hour": 16, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 880}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.755, "sample_count": 2117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1924336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:53:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T05:53:29+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T05:53:29+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T05:53:29.058995+09:00", "mtime_epoch": 1783284809.058995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:53:29+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9543.582, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.173, "pushed": 7.173, "read": 7.173}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.592, "count": 214, "max_ms": 27.757, "p95_ms": 22.063}, "gstreamer_push": {"avg_ms": 1.114, "count": 215, "max_ms": 1.641, "p95_ms": 1.198}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.473, "count": 214, "max_ms": 64.903, "p95_ms": 62.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.177, "count": 214, "max_ms": 86.264, "p95_ms": 82.548}, "yolo11_640_fire_11class": {"avg_ms": 51.587, "count": 214, "max_ms": 67.609, "p95_ms": 65.187}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.735, "count": 214, "max_ms": 83.481, "p95_ms": 80.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.362, "count": 215, "max_ms": 175.532, "p95_ms": 146.679}, "read_interval": {"avg_ms": 139.84, "count": 215, "max_ms": 186.558, "p95_ms": 158.557}, "triton_wait": {"avg_ms": 64.288, "count": 214, "max_ms": 86.276, "p95_ms": 82.663}, "visualize": {"avg_ms": 4.781, "count": 214, "max_ms": 6.133, "p95_ms": 5.801}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16036}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16282.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:53:29+09:00", "last_error_summary": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 790, "h264_decode_error_count": 1938, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:53:29+09:00", "last_error_summary": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "last_exit_at": "2026-07-06T05:53:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.787, "last_started_at": "2026-07-06T05:50:30+09:00", "pid": 147291, "process": {"exists": false, "pid": 147291}, "restart_count": 879, "restart_count_per_hour": 16, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 880}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.755, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1924337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:53:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T05:53:29+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-06T05:53:29+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-06T05:53:29.058995+09:00", "mtime_epoch": 1783284809.058995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:53:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9543.582, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.185, "pushed": 7.185, "read": 7.151}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.587, "count": 215, "max_ms": 27.757, "p95_ms": 22.063}, "gstreamer_push": {"avg_ms": 1.113, "count": 215, "max_ms": 1.641, "p95_ms": 1.198}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.535, "count": 215, "max_ms": 64.903, "p95_ms": 62.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.188, "count": 215, "max_ms": 86.264, "p95_ms": 82.548}, "yolo11_640_fire_11class": {"avg_ms": 51.612, "count": 215, "max_ms": 67.609, "p95_ms": 65.187}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.792, "count": 215, "max_ms": 83.481, "p95_ms": 80.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.318, "count": 215, "max_ms": 175.532, "p95_ms": 146.679}, "read_interval": {"avg_ms": 139.779, "count": 214, "max_ms": 186.558, "p95_ms": 158.557}, "triton_wait": {"avg_ms": 64.324, "count": 215, "max_ms": 86.276, "p95_ms": 82.663}, "visualize": {"avg_ms": 4.781, "count": 215, "max_ms": 6.133, "p95_ms": 5.801}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16036}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16282.488, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 790, "h264_decode_error_count": 1938, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:53:29+09:00", "last_error_summary": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "last_exit_at": "2026-07-06T05:53:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.792, "last_started_at": "2026-07-06T05:50:30+09:00", "pid": 147291, "process": {"exists": false, "pid": 147291}, "restart_count": 879, "restart_count_per_hour": 16, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 880}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.755, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1924337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:53:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T05:53:29+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-06T05:53:29+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-06T05:53:29.058995+09:00", "mtime_epoch": 1783284809.058995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:53:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9543.582, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.184, "pushed": 7.184, "read": 7.15}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.587, "count": 215, "max_ms": 27.757, "p95_ms": 22.063}, "gstreamer_push": {"avg_ms": 1.113, "count": 215, "max_ms": 1.641, "p95_ms": 1.198}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.535, "count": 215, "max_ms": 64.903, "p95_ms": 62.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.188, "count": 215, "max_ms": 86.264, "p95_ms": 82.548}, "yolo11_640_fire_11class": {"avg_ms": 51.612, "count": 215, "max_ms": 67.609, "p95_ms": 65.187}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.792, "count": 215, "max_ms": 83.481, "p95_ms": 80.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.318, "count": 215, "max_ms": 175.532, "p95_ms": 146.679}, "read_interval": {"avg_ms": 139.779, "count": 214, "max_ms": 186.558, "p95_ms": 158.557}, "triton_wait": {"avg_ms": 64.324, "count": 215, "max_ms": 86.276, "p95_ms": 82.663}, "visualize": {"avg_ms": 4.781, "count": 215, "max_ms": 6.133, "p95_ms": 5.801}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16036}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16391.012, "limit_mb": 8796093022207.996}, "detail": {"pid": 147398, "restart_count": 880, "start_count": 881}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 790, "h264_decode_error_count": 1938, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:53:29+09:00", "last_error_summary": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "last_exit_at": "2026-07-06T05:53:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:53:32+09:00", "pid": 147398, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.5, "name": "ffmpeg", "pid": 147398, "rss_mb": 139.5, "state": "S (sleeping)", "threads": 106}, "restart_count": 880, "restart_count_per_hour": 17, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 881}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.85, "sample_count": 2082, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1924337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:53:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.956, "last_frame_pushed_at": "2026-07-06T05:53:29+09:00", "last_frame_read_age_sec": 5.089, "last_frame_read_at": "2026-07-06T05:53:29+09:00", "mjpeg": {"age_sec": 4.939, "exists": true, "mtime": "2026-07-06T05:53:29.202997+09:00", "mtime_epoch": 1783284809.2029967, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:53:34+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9543.582, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.988, "pushed": 5.988, "read": 5.954}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.605, "count": 179, "max_ms": 27.757, "p95_ms": 21.588}, "gstreamer_push": {"avg_ms": 1.11, "count": 179, "max_ms": 1.641, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.422, "count": 179, "max_ms": 64.177, "p95_ms": 62.683}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.319, "count": 179, "max_ms": 86.264, "p95_ms": 82.548}, "yolo11_640_fire_11class": {"avg_ms": 51.657, "count": 179, "max_ms": 67.609, "p95_ms": 65.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.584, "count": 179, "max_ms": 83.481, "p95_ms": 80.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.586, "count": 179, "max_ms": 175.532, "p95_ms": 146.768}, "read_interval": {"avg_ms": 140.025, "count": 178, "max_ms": 186.558, "p95_ms": 158.874}, "triton_wait": {"avg_ms": 64.253, "count": 179, "max_ms": 86.276, "p95_ms": 82.663}, "visualize": {"avg_ms": 4.788, "count": 179, "max_ms": 6.133, "p95_ms": 5.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16283.5, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:53:29+09:00", "last_error_summary": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 790, "h264_decode_error_count": 1938, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:53:29+09:00", "last_error_summary": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "last_exit_at": "2026-07-06T05:55:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.783, "last_started_at": "2026-07-06T05:53:32+09:00", "pid": 147398, "process": {"exists": false, "pid": 147398}, "restart_count": 880, "restart_count_per_hour": 16, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 881}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 204.428, "sample_count": 1883, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1925145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:55:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T05:55:56+09:00", "last_frame_read_age_sec": 0.185, "last_frame_read_at": "2026-07-06T05:55:56+09:00", "mjpeg": {"age_sec": 0.178, "exists": true, "mtime": "2026-07-06T05:55:56.744889+09:00", "mtime_epoch": 1783284956.7448893, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158464}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:55:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.098, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.682, "pushed": 4.682, "read": 4.648}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.938, "count": 139, "max_ms": 32.326, "p95_ms": 27.439}, "gstreamer_push": {"avg_ms": 0.859, "count": 140, "max_ms": 1.155, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.418, "count": 139, "max_ms": 64.864, "p95_ms": 60.587}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.192, "count": 139, "max_ms": 87.411, "p95_ms": 81.36}, "yolo11_640_fire_11class": {"avg_ms": 48.303, "count": 139, "max_ms": 68.078, "p95_ms": 61.951}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.342, "count": 139, "max_ms": 83.608, "p95_ms": 81.576}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.826, "count": 140, "max_ms": 241.675, "p95_ms": 230.675}, "read_interval": {"avg_ms": 215.21, "count": 139, "max_ms": 253.387, "p95_ms": 241.838}, "triton_wait": {"avg_ms": 64.644, "count": 139, "max_ms": 87.422, "p95_ms": 82.2}, "visualize": {"avg_ms": 6.852, "count": 139, "max_ms": 9.999, "p95_ms": 7.984}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16283.492, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 790, "h264_decode_error_count": 1938, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:53:29+09:00", "last_error_summary": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "last_exit_at": "2026-07-06T05:55:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.786, "last_started_at": "2026-07-06T05:53:32+09:00", "pid": 147398, "process": {"exists": false, "pid": 147398}, "restart_count": 880, "restart_count_per_hour": 16, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 881}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 204.428, "sample_count": 1883, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1925145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:55:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T05:55:56+09:00", "last_frame_read_age_sec": 0.189, "last_frame_read_at": "2026-07-06T05:55:56+09:00", "mjpeg": {"age_sec": 0.182, "exists": true, "mtime": "2026-07-06T05:55:56.744889+09:00", "mtime_epoch": 1783284956.7448893, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158464}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:55:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.098, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.681, "pushed": 4.681, "read": 4.648}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.938, "count": 139, "max_ms": 32.326, "p95_ms": 27.439}, "gstreamer_push": {"avg_ms": 0.859, "count": 140, "max_ms": 1.155, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.418, "count": 139, "max_ms": 64.864, "p95_ms": 60.587}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.192, "count": 139, "max_ms": 87.411, "p95_ms": 81.36}, "yolo11_640_fire_11class": {"avg_ms": 48.303, "count": 139, "max_ms": 68.078, "p95_ms": 61.951}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.342, "count": 139, "max_ms": 83.608, "p95_ms": 81.576}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.826, "count": 140, "max_ms": 241.675, "p95_ms": 230.675}, "read_interval": {"avg_ms": 215.21, "count": 139, "max_ms": 253.387, "p95_ms": 241.838}, "triton_wait": {"avg_ms": 64.644, "count": 139, "max_ms": 87.422, "p95_ms": 82.2}, "visualize": {"avg_ms": 6.852, "count": 139, "max_ms": 9.999, "p95_ms": 7.984}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16394.41, "limit_mb": 8796093022207.996}, "detail": {"pid": 147505, "restart_count": 881, "start_count": 882}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 790, "h264_decode_error_count": 1938, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:53:29+09:00", "last_error_summary": "[h264 @ 0x55bc20048dc0] error while decoding MB 67 21, bytestream -8", "last_exit_at": "2026-07-06T05:55:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:55:59+09:00", "pid": 147505, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.551, "name": "ffmpeg", "pid": 147505, "rss_mb": 141.551, "state": "S (sleeping)", "threads": 106}, "restart_count": 881, "restart_count_per_hour": 17, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 882}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 204.435, "sample_count": 1854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1925145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:55:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.294, "last_frame_pushed_at": "2026-07-06T05:55:56+09:00", "last_frame_read_age_sec": 4.475, "last_frame_read_at": "2026-07-06T05:55:56+09:00", "mjpeg": {"age_sec": 4.281, "exists": true, "mtime": "2026-07-06T05:55:56.932892+09:00", "mtime_epoch": 1783284956.9328918, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158600}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:56:01+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.098, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.98, "pushed": 3.98, "read": 3.946}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.0, "count": 118, "max_ms": 32.326, "p95_ms": 27.321}, "gstreamer_push": {"avg_ms": 0.857, "count": 119, "max_ms": 1.155, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.359, "count": 118, "max_ms": 62.267, "p95_ms": 60.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.038, "count": 118, "max_ms": 87.411, "p95_ms": 81.36}, "yolo11_640_fire_11class": {"avg_ms": 48.019, "count": 118, "max_ms": 62.805, "p95_ms": 61.411}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.994, "count": 118, "max_ms": 83.608, "p95_ms": 80.781}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.859, "count": 119, "max_ms": 241.675, "p95_ms": 233.68}, "read_interval": {"avg_ms": 217.292, "count": 118, "max_ms": 253.387, "p95_ms": 243.476}, "triton_wait": {"avg_ms": 64.481, "count": 118, "max_ms": 87.422, "p95_ms": 81.588}, "visualize": {"avg_ms": 6.87, "count": 118, "max_ms": 9.999, "p95_ms": 7.984}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16051}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16285.465, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56139bf8f240] error while decoding MB 9 30, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 791, "h264_decode_error_count": 1939, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:58:44+09:00", "last_error_summary": "[h264 @ 0x56139bf8f240] error while decoding MB 9 30, bytestream -9", "last_exit_at": "2026-07-06T05:55:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 165.335, "last_started_at": "2026-07-06T05:55:59+09:00", "pid": 147505, "process": {"exists": false, "pid": 147505}, "restart_count": 881, "restart_count_per_hour": 17, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 882}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.043, "sample_count": 1784, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1926193, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:58:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T05:58:44+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T05:58:44+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T05:58:44.507066+09:00", "mtime_epoch": 1783285124.5070662, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154228}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:58:44+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.355, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.103, "pushed": 5.103, "read": 5.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.858, "count": 152, "max_ms": 31.405, "p95_ms": 25.216}, "gstreamer_push": {"avg_ms": 0.89, "count": 153, "max_ms": 1.093, "p95_ms": 0.98}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.574, "count": 152, "max_ms": 64.613, "p95_ms": 61.404}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.865, "count": 152, "max_ms": 87.34, "p95_ms": 83.178}, "yolo11_640_fire_11class": {"avg_ms": 48.517, "count": 152, "max_ms": 69.397, "p95_ms": 65.31}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.071, "count": 152, "max_ms": 87.0, "p95_ms": 83.505}}, "truncated_count": 0}, "processing_total": {"avg_ms": 185.068, "count": 153, "max_ms": 235.954, "p95_ms": 221.346}, "read_interval": {"avg_ms": 196.588, "count": 153, "max_ms": 366.776, "p95_ms": 231.211}, "triton_wait": {"avg_ms": 62.347, "count": 152, "max_ms": 87.352, "p95_ms": 83.849}, "visualize": {"avg_ms": 7.388, "count": 152, "max_ms": 9.357, "p95_ms": 8.747}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16051}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16285.0, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T05:58:44+09:00", "last_error_summary": "[h264 @ 0x56139bf8f240] error while decoding MB 9 30, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 791, "h264_decode_error_count": 1939, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:58:44+09:00", "last_error_summary": "[h264 @ 0x56139bf8f240] error while decoding MB 9 30, bytestream -9", "last_exit_at": "2026-07-06T05:58:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 165.46, "last_started_at": "2026-07-06T05:55:59+09:00", "pid": 147505, "process": {"exists": false, "pid": 147505}, "restart_count": 881, "restart_count_per_hour": 17, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 882}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.043, "sample_count": 1785, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1926194, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:58:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T05:58:44+09:00", "last_frame_read_age_sec": 0.175, "last_frame_read_at": "2026-07-06T05:58:44+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-06T05:58:44.507066+09:00", "mtime_epoch": 1783285124.5070662, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154228}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:58:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.355, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.108, "pushed": 5.108, "read": 5.075}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.881, "count": 152, "max_ms": 31.405, "p95_ms": 25.216}, "gstreamer_push": {"avg_ms": 0.89, "count": 153, "max_ms": 1.093, "p95_ms": 0.98}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.507, "count": 152, "max_ms": 64.613, "p95_ms": 61.404}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.832, "count": 152, "max_ms": 87.34, "p95_ms": 83.178}, "yolo11_640_fire_11class": {"avg_ms": 48.46, "count": 152, "max_ms": 69.397, "p95_ms": 65.31}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.047, "count": 152, "max_ms": 87.0, "p95_ms": 83.505}}, "truncated_count": 0}, "processing_total": {"avg_ms": 185.238, "count": 153, "max_ms": 235.954, "p95_ms": 221.346}, "read_interval": {"avg_ms": 196.862, "count": 152, "max_ms": 366.776, "p95_ms": 231.211}, "triton_wait": {"avg_ms": 62.314, "count": 152, "max_ms": 87.352, "p95_ms": 83.849}, "visualize": {"avg_ms": 7.398, "count": 152, "max_ms": 9.357, "p95_ms": 8.747}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16051}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16284.992, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 791, "h264_decode_error_count": 1939, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:58:44+09:00", "last_error_summary": "[h264 @ 0x56139bf8f240] error while decoding MB 9 30, bytestream -9", "last_exit_at": "2026-07-06T05:58:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 165.464, "last_started_at": "2026-07-06T05:55:59+09:00", "pid": 147505, "process": {"exists": false, "pid": 147505}, "restart_count": 881, "restart_count_per_hour": 17, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 882}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.043, "sample_count": 1785, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1926194, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:58:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T05:58:44+09:00", "last_frame_read_age_sec": 0.179, "last_frame_read_at": "2026-07-06T05:58:44+09:00", "mjpeg": {"age_sec": 0.169, "exists": true, "mtime": "2026-07-06T05:58:44.507066+09:00", "mtime_epoch": 1783285124.5070662, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154228}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:58:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.355, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.108, "pushed": 5.108, "read": 5.074}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.881, "count": 152, "max_ms": 31.405, "p95_ms": 25.216}, "gstreamer_push": {"avg_ms": 0.89, "count": 153, "max_ms": 1.093, "p95_ms": 0.98}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.507, "count": 152, "max_ms": 64.613, "p95_ms": 61.404}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.832, "count": 152, "max_ms": 87.34, "p95_ms": 83.178}, "yolo11_640_fire_11class": {"avg_ms": 48.46, "count": 152, "max_ms": 69.397, "p95_ms": 65.31}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.047, "count": 152, "max_ms": 87.0, "p95_ms": 83.505}}, "truncated_count": 0}, "processing_total": {"avg_ms": 185.238, "count": 153, "max_ms": 235.954, "p95_ms": 221.346}, "read_interval": {"avg_ms": 196.862, "count": 152, "max_ms": 366.776, "p95_ms": 231.211}, "triton_wait": {"avg_ms": 62.314, "count": 152, "max_ms": 87.352, "p95_ms": 83.849}, "visualize": {"avg_ms": 7.398, "count": 152, "max_ms": 9.357, "p95_ms": 8.747}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 336, "tracking_summary": 16051}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16390.598, "limit_mb": 8796093022207.996}, "detail": {"pid": 147612, "restart_count": 882, "start_count": 883}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 791, "h264_decode_error_count": 1939, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T05:58:44+09:00", "last_error_summary": "[h264 @ 0x56139bf8f240] error while decoding MB 9 30, bytestream -9", "last_exit_at": "2026-07-06T05:58:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T05:58:47+09:00", "pid": 147612, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.754, "name": "ffmpeg", "pid": 147612, "rss_mb": 136.754, "state": "S (sleeping)", "threads": 106}, "restart_count": 882, "restart_count_per_hour": 17, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 883}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.181, "sample_count": 1753, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1926194, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T05:58:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.944, "last_frame_pushed_at": "2026-07-06T05:58:44+09:00", "last_frame_read_age_sec": 5.116, "last_frame_read_at": "2026-07-06T05:58:44+09:00", "mjpeg": {"age_sec": 4.926, "exists": true, "mtime": "2026-07-06T05:58:44.687069+09:00", "mtime_epoch": 1783285124.6870687, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154502}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T05:58:49+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.355, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.221, "pushed": 4.221, "read": 4.187}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.029, "count": 126, "max_ms": 31.405, "p95_ms": 25.33}, "gstreamer_push": {"avg_ms": 0.882, "count": 126, "max_ms": 1.093, "p95_ms": 0.951}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.07, "count": 126, "max_ms": 64.613, "p95_ms": 61.455}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.419, "count": 126, "max_ms": 87.34, "p95_ms": 77.849}, "yolo11_640_fire_11class": {"avg_ms": 47.759, "count": 126, "max_ms": 69.397, "p95_ms": 64.471}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.367, "count": 126, "max_ms": 86.11, "p95_ms": 82.646}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.339, "count": 126, "max_ms": 235.954, "p95_ms": 221.432}, "read_interval": {"avg_ms": 199.217, "count": 125, "max_ms": 366.776, "p95_ms": 234.112}, "triton_wait": {"avg_ms": 61.139, "count": 126, "max_ms": 87.352, "p95_ms": 83.517}, "visualize": {"avg_ms": 7.521, "count": 126, "max_ms": 9.357, "p95_ms": 8.773}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.853, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16056}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16291.688, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5560a8ca4e40] error while decoding MB 68 5, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 792, "h264_decode_error_count": 1940, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:00:58+09:00", "last_error_summary": "[h264 @ 0x5560a8ca4e40] error while decoding MB 68 5, bytestream -8", "last_exit_at": "2026-07-06T05:58:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.889, "last_started_at": "2026-07-06T05:58:47+09:00", "pid": 147612, "process": {"exists": false, "pid": 147612}, "restart_count": 882, "restart_count_per_hour": 17, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 883}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.432, "sample_count": 1624, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1926769, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:00:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.042, "last_frame_pushed_at": "2026-07-06T06:00:58+09:00", "last_frame_read_age_sec": 0.034, "last_frame_read_at": "2026-07-06T06:00:58+09:00", "mjpeg": {"age_sec": 0.029, "exists": true, "mtime": "2026-07-06T06:00:58.472819+09:00", "mtime_epoch": 1783285258.4728189, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:00:58+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.871, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.244, "pushed": 4.244, "read": 4.244}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.427, "count": 126, "max_ms": 35.225, "p95_ms": 28.98}, "gstreamer_push": {"avg_ms": 0.859, "count": 127, "max_ms": 1.339, "p95_ms": 0.92}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.682, "count": 126, "max_ms": 73.276, "p95_ms": 63.511}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.286, "count": 126, "max_ms": 91.164, "p95_ms": 84.692}, "yolo11_640_fire_11class": {"avg_ms": 51.235, "count": 126, "max_ms": 74.421, "p95_ms": 67.489}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.207, "count": 126, "max_ms": 89.085, "p95_ms": 83.874}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.198, "count": 127, "max_ms": 918.862, "p95_ms": 274.534}, "read_interval": {"avg_ms": 237.11, "count": 127, "max_ms": 929.925, "p95_ms": 284.197}, "triton_wait": {"avg_ms": 66.713, "count": 126, "max_ms": 91.177, "p95_ms": 84.714}, "visualize": {"avg_ms": 9.822, "count": 126, "max_ms": 14.862, "p95_ms": 13.778}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16056}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16291.016, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:00:58+09:00", "last_error_summary": "[h264 @ 0x5560a8ca4e40] error while decoding MB 68 5, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 792, "h264_decode_error_count": 1940, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:00:58+09:00", "last_error_summary": "[h264 @ 0x5560a8ca4e40] error while decoding MB 68 5, bytestream -8", "last_exit_at": "2026-07-06T06:00:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.045, "last_started_at": "2026-07-06T05:58:47+09:00", "pid": 147612, "process": {"exists": false, "pid": 147612}, "restart_count": 882, "restart_count_per_hour": 17, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 883}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.432, "sample_count": 1625, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1926770, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:00:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:00:58+09:00", "last_frame_read_age_sec": 0.19, "last_frame_read_at": "2026-07-06T06:00:58+09:00", "mjpeg": {"age_sec": 0.185, "exists": true, "mtime": "2026-07-06T06:00:58.472819+09:00", "mtime_epoch": 1783285258.4728189, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:00:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.871, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.254, "pushed": 4.254, "read": 4.22}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.394, "count": 127, "max_ms": 35.225, "p95_ms": 28.98}, "gstreamer_push": {"avg_ms": 0.859, "count": 127, "max_ms": 1.339, "p95_ms": 0.92}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.727, "count": 127, "max_ms": 73.276, "p95_ms": 63.511}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.389, "count": 127, "max_ms": 91.164, "p95_ms": 84.692}, "yolo11_640_fire_11class": {"avg_ms": 51.287, "count": 127, "max_ms": 74.421, "p95_ms": 67.489}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.289, "count": 127, "max_ms": 89.085, "p95_ms": 83.874}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.926, "count": 127, "max_ms": 918.862, "p95_ms": 274.534}, "read_interval": {"avg_ms": 237.172, "count": 126, "max_ms": 929.925, "p95_ms": 284.197}, "triton_wait": {"avg_ms": 66.805, "count": 127, "max_ms": 91.177, "p95_ms": 84.714}, "visualize": {"avg_ms": 9.794, "count": 127, "max_ms": 14.862, "p95_ms": 13.778}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16056}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16290.938, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 792, "h264_decode_error_count": 1940, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:00:58+09:00", "last_error_summary": "[h264 @ 0x5560a8ca4e40] error while decoding MB 68 5, bytestream -8", "last_exit_at": "2026-07-06T06:00:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.049, "last_started_at": "2026-07-06T05:58:47+09:00", "pid": 147612, "process": {"exists": false, "pid": 147612}, "restart_count": 882, "restart_count_per_hour": 17, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 883}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.432, "sample_count": 1625, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1926770, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:00:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T06:00:58+09:00", "last_frame_read_age_sec": 0.194, "last_frame_read_at": "2026-07-06T06:00:58+09:00", "mjpeg": {"age_sec": 0.188, "exists": true, "mtime": "2026-07-06T06:00:58.472819+09:00", "mtime_epoch": 1783285258.4728189, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:00:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.871, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.253, "pushed": 4.253, "read": 4.22}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.394, "count": 127, "max_ms": 35.225, "p95_ms": 28.98}, "gstreamer_push": {"avg_ms": 0.859, "count": 127, "max_ms": 1.339, "p95_ms": 0.92}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.727, "count": 127, "max_ms": 73.276, "p95_ms": 63.511}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.389, "count": 127, "max_ms": 91.164, "p95_ms": 84.692}, "yolo11_640_fire_11class": {"avg_ms": 51.287, "count": 127, "max_ms": 74.421, "p95_ms": 67.489}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.289, "count": 127, "max_ms": 89.085, "p95_ms": 83.874}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.926, "count": 127, "max_ms": 918.862, "p95_ms": 274.534}, "read_interval": {"avg_ms": 237.172, "count": 126, "max_ms": 929.925, "p95_ms": 284.197}, "triton_wait": {"avg_ms": 66.805, "count": 127, "max_ms": 91.177, "p95_ms": 84.714}, "visualize": {"avg_ms": 9.794, "count": 127, "max_ms": 14.862, "p95_ms": 13.778}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16056}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16399.285, "limit_mb": 8796093022207.996}, "detail": {"pid": 147779, "restart_count": 883, "start_count": 884}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 792, "h264_decode_error_count": 1940, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:00:58+09:00", "last_error_summary": "[h264 @ 0x5560a8ca4e40] error while decoding MB 68 5, bytestream -8", "last_exit_at": "2026-07-06T06:01:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:01:01+09:00", "pid": 147779, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.188, "name": "ffmpeg", "pid": 147779, "rss_mb": 139.188, "state": "S (sleeping)", "threads": 106}, "restart_count": 883, "restart_count_per_hour": 18, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 884}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.572, "sample_count": 1610, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1926770, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:00:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.499, "last_frame_pushed_at": "2026-07-06T06:00:58+09:00", "last_frame_read_age_sec": 4.686, "last_frame_read_at": "2026-07-06T06:00:58+09:00", "mjpeg": {"age_sec": 4.485, "exists": true, "mtime": "2026-07-06T06:00:58.668822+09:00", "mtime_epoch": 1783285258.6688216, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144671}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:01:03+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9544.871, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.593, "pushed": 3.593, "read": 3.559}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.267, "count": 107, "max_ms": 35.225, "p95_ms": 29.108}, "gstreamer_push": {"avg_ms": 0.862, "count": 107, "max_ms": 1.339, "p95_ms": 0.928}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.924, "count": 107, "max_ms": 73.276, "p95_ms": 63.511}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.634, "count": 107, "max_ms": 91.164, "p95_ms": 82.723}, "yolo11_640_fire_11class": {"avg_ms": 50.893, "count": 107, "max_ms": 74.421, "p95_ms": 67.489}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.487, "count": 107, "max_ms": 89.085, "p95_ms": 80.817}}, "truncated_count": 0}, "processing_total": {"avg_ms": 228.488, "count": 107, "max_ms": 918.862, "p95_ms": 274.704}, "read_interval": {"avg_ms": 238.829, "count": 106, "max_ms": 929.925, "p95_ms": 285.834}, "triton_wait": {"avg_ms": 66.151, "count": 107, "max_ms": 91.177, "p95_ms": 82.735}, "visualize": {"avg_ms": 10.031, "count": 107, "max_ms": 14.862, "p95_ms": 13.841}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.784, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16292.184, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5636df0ff340] error while decoding MB 19 42, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 793, "h264_decode_error_count": 1941, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:03:12+09:00", "last_error_summary": "[h264 @ 0x5636df0ff340] error while decoding MB 19 42, bytestream -5", "last_exit_at": "2026-07-06T06:01:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.766, "last_started_at": "2026-07-06T06:01:01+09:00", "pid": 147779, "process": {"exists": false, "pid": 147779}, "restart_count": 883, "restart_count_per_hour": 18, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 884}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.064, "sample_count": 1311, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1927342, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:03:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T06:03:12+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T06:03:12+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T06:03:12.878587+09:00", "mtime_epoch": 1783285392.878587, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 146456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:03:12+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.129, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.968, "pushed": 3.968, "read": 4.001}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.498, "count": 119, "max_ms": 40.44, "p95_ms": 30.979}, "gstreamer_push": {"avg_ms": 0.861, "count": 119, "max_ms": 1.292, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.843, "count": 119, "max_ms": 63.169, "p95_ms": 61.829}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.541, "count": 119, "max_ms": 88.836, "p95_ms": 84.748}, "yolo11_640_fire_11class": {"avg_ms": 49.86, "count": 119, "max_ms": 65.547, "p95_ms": 64.901}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.15, "count": 119, "max_ms": 87.371, "p95_ms": 82.744}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.743, "count": 119, "max_ms": 287.55, "p95_ms": 275.248}, "read_interval": {"avg_ms": 251.543, "count": 120, "max_ms": 297.282, "p95_ms": 283.756}, "triton_wait": {"avg_ms": 67.303, "count": 119, "max_ms": 88.847, "p95_ms": 85.211}, "visualize": {"avg_ms": 9.557, "count": 119, "max_ms": 14.678, "p95_ms": 10.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16291.75, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:03:12+09:00", "last_error_summary": "[h264 @ 0x5636df0ff340] error while decoding MB 19 42, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 793, "h264_decode_error_count": 1941, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:03:12+09:00", "last_error_summary": "[h264 @ 0x5636df0ff340] error while decoding MB 19 42, bytestream -5", "last_exit_at": "2026-07-06T06:03:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.976, "last_started_at": "2026-07-06T06:01:01+09:00", "pid": 147779, "process": {"exists": false, "pid": 147779}, "restart_count": 883, "restart_count_per_hour": 18, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 884}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.064, "sample_count": 1311, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1927343, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:03:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T06:03:13+09:00", "last_frame_read_age_sec": 0.261, "last_frame_read_at": "2026-07-06T06:03:12+09:00", "mjpeg": {"age_sec": 0.25, "exists": true, "mtime": "2026-07-06T06:03:12.878587+09:00", "mtime_epoch": 1783285392.878587, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 146456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:03:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.129, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.002, "pushed": 4.002, "read": 3.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.509, "count": 119, "max_ms": 40.44, "p95_ms": 30.979}, "gstreamer_push": {"avg_ms": 0.861, "count": 120, "max_ms": 1.292, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.841, "count": 119, "max_ms": 63.169, "p95_ms": 61.829}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.509, "count": 119, "max_ms": 88.836, "p95_ms": 84.748}, "yolo11_640_fire_11class": {"avg_ms": 49.846, "count": 119, "max_ms": 65.547, "p95_ms": 64.901}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.155, "count": 119, "max_ms": 87.371, "p95_ms": 82.744}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.853, "count": 120, "max_ms": 287.55, "p95_ms": 274.234}, "read_interval": {"avg_ms": 251.615, "count": 119, "max_ms": 297.282, "p95_ms": 284.874}, "triton_wait": {"avg_ms": 67.271, "count": 119, "max_ms": 88.847, "p95_ms": 85.211}, "visualize": {"avg_ms": 9.572, "count": 119, "max_ms": 14.678, "p95_ms": 10.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16291.695, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 793, "h264_decode_error_count": 1941, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:03:12+09:00", "last_error_summary": "[h264 @ 0x5636df0ff340] error while decoding MB 19 42, bytestream -5", "last_exit_at": "2026-07-06T06:03:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.98, "last_started_at": "2026-07-06T06:01:01+09:00", "pid": 147779, "process": {"exists": false, "pid": 147779}, "restart_count": 883, "restart_count_per_hour": 18, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 884}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.064, "sample_count": 1311, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1927343, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:03:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T06:03:13+09:00", "last_frame_read_age_sec": 0.264, "last_frame_read_at": "2026-07-06T06:03:12+09:00", "mjpeg": {"age_sec": 0.254, "exists": true, "mtime": "2026-07-06T06:03:12.878587+09:00", "mtime_epoch": 1783285392.878587, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 146456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:03:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.129, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.001, "pushed": 4.001, "read": 3.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.509, "count": 119, "max_ms": 40.44, "p95_ms": 30.979}, "gstreamer_push": {"avg_ms": 0.861, "count": 120, "max_ms": 1.292, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.841, "count": 119, "max_ms": 63.169, "p95_ms": 61.829}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.509, "count": 119, "max_ms": 88.836, "p95_ms": 84.748}, "yolo11_640_fire_11class": {"avg_ms": 49.846, "count": 119, "max_ms": 65.547, "p95_ms": 64.901}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.155, "count": 119, "max_ms": 87.371, "p95_ms": 82.744}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.853, "count": 120, "max_ms": 287.55, "p95_ms": 274.234}, "read_interval": {"avg_ms": 251.615, "count": 119, "max_ms": 297.282, "p95_ms": 284.874}, "triton_wait": {"avg_ms": 67.271, "count": 119, "max_ms": 88.847, "p95_ms": 85.211}, "visualize": {"avg_ms": 9.572, "count": 119, "max_ms": 14.678, "p95_ms": 10.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16400.043, "limit_mb": 8796093022207.996}, "detail": {"pid": 147886, "restart_count": 884, "start_count": 885}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 793, "h264_decode_error_count": 1941, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:03:12+09:00", "last_error_summary": "[h264 @ 0x5636df0ff340] error while decoding MB 19 42, bytestream -5", "last_exit_at": "2026-07-06T06:03:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:03:15+09:00", "pid": 147886, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.203, "name": "ffmpeg", "pid": 147886, "rss_mb": 139.203, "state": "S (sleeping)", "threads": 106}, "restart_count": 884, "restart_count_per_hour": 19, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 885}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.724, "sample_count": 1289, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1927343, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:03:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.031, "last_frame_pushed_at": "2026-07-06T06:03:13+09:00", "last_frame_read_age_sec": 4.287, "last_frame_read_at": "2026-07-06T06:03:12+09:00", "mjpeg": {"age_sec": 4.013, "exists": true, "mtime": "2026-07-06T06:03:13.142591+09:00", "mtime_epoch": 1783285393.1425905, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 146473}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:03:17+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.129, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.45, "pushed": 3.45, "read": 3.416}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.669, "count": 103, "max_ms": 40.44, "p95_ms": 30.979}, "gstreamer_push": {"avg_ms": 0.864, "count": 103, "max_ms": 1.292, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.802, "count": 103, "max_ms": 63.169, "p95_ms": 61.821}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.185, "count": 103, "max_ms": 88.836, "p95_ms": 84.748}, "yolo11_640_fire_11class": {"avg_ms": 49.695, "count": 103, "max_ms": 65.277, "p95_ms": 64.42}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.931, "count": 103, "max_ms": 87.371, "p95_ms": 82.433}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.222, "count": 103, "max_ms": 287.55, "p95_ms": 275.248}, "read_interval": {"avg_ms": 252.999, "count": 102, "max_ms": 297.282, "p95_ms": 284.874}, "triton_wait": {"avg_ms": 66.97, "count": 103, "max_ms": 88.847, "p95_ms": 84.981}, "visualize": {"avg_ms": 9.612, "count": 103, "max_ms": 14.678, "p95_ms": 10.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.857, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16064}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16293.555, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e757fc4500] error while decoding MB 54 9, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 794, "h264_decode_error_count": 1942, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:05:06+09:00", "last_error_summary": "[h264 @ 0x55e757fc4500] error while decoding MB 54 9, bytestream -7", "last_exit_at": "2026-07-06T06:03:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.459, "last_started_at": "2026-07-06T06:03:15+09:00", "pid": 147886, "process": {"exists": false, "pid": 147886}, "restart_count": 884, "restart_count_per_hour": 19, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 885}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.139, "sample_count": 1240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1927786, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:05:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T06:05:06+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T06:05:06+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T06:05:06.564088+09:00", "mtime_epoch": 1783285506.564088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 150809}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:05:06+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.129, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.035, "pushed": 4.035, "read": 4.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.941, "count": 120, "max_ms": 29.198, "p95_ms": 28.044}, "gstreamer_push": {"avg_ms": 0.887, "count": 121, "max_ms": 0.97, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.086, "count": 120, "max_ms": 65.227, "p95_ms": 61.843}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.607, "count": 120, "max_ms": 91.041, "p95_ms": 87.729}, "yolo11_640_fire_11class": {"avg_ms": 49.81, "count": 120, "max_ms": 66.185, "p95_ms": 64.127}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.84, "count": 120, "max_ms": 88.805, "p95_ms": 84.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.497, "count": 121, "max_ms": 302.719, "p95_ms": 284.437}, "read_interval": {"avg_ms": 249.177, "count": 121, "max_ms": 313.728, "p95_ms": 294.373}, "triton_wait": {"avg_ms": 68.746, "count": 120, "max_ms": 91.053, "p95_ms": 87.742}, "visualize": {"avg_ms": 9.345, "count": 120, "max_ms": 11.698, "p95_ms": 11.237}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16064}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16292.605, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:05:06+09:00", "last_error_summary": "[h264 @ 0x55e757fc4500] error while decoding MB 54 9, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 794, "h264_decode_error_count": 1942, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:05:06+09:00", "last_error_summary": "[h264 @ 0x55e757fc4500] error while decoding MB 54 9, bytestream -7", "last_exit_at": "2026-07-06T06:05:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.66, "last_started_at": "2026-07-06T06:03:15+09:00", "pid": 147886, "process": {"exists": false, "pid": 147886}, "restart_count": 884, "restart_count_per_hour": 19, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 885}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.139, "sample_count": 1240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1927787, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:05:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T06:05:06+09:00", "last_frame_read_age_sec": 0.256, "last_frame_read_at": "2026-07-06T06:05:06+09:00", "mjpeg": {"age_sec": 0.251, "exists": true, "mtime": "2026-07-06T06:05:06.564088+09:00", "mtime_epoch": 1783285506.564088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 150809}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:05:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.129, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.04, "pushed": 4.04, "read": 4.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.953, "count": 120, "max_ms": 29.198, "p95_ms": 28.044}, "gstreamer_push": {"avg_ms": 0.887, "count": 121, "max_ms": 0.97, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.1, "count": 120, "max_ms": 65.227, "p95_ms": 61.843}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.639, "count": 120, "max_ms": 91.041, "p95_ms": 87.729}, "yolo11_640_fire_11class": {"avg_ms": 49.725, "count": 120, "max_ms": 66.185, "p95_ms": 64.127}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.726, "count": 120, "max_ms": 88.805, "p95_ms": 84.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.777, "count": 121, "max_ms": 302.719, "p95_ms": 284.437}, "read_interval": {"avg_ms": 249.381, "count": 120, "max_ms": 313.728, "p95_ms": 294.373}, "triton_wait": {"avg_ms": 68.668, "count": 120, "max_ms": 91.053, "p95_ms": 87.742}, "visualize": {"avg_ms": 9.352, "count": 120, "max_ms": 11.698, "p95_ms": 11.237}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16064}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16292.598, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 794, "h264_decode_error_count": 1942, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:05:06+09:00", "last_error_summary": "[h264 @ 0x55e757fc4500] error while decoding MB 54 9, bytestream -7", "last_exit_at": "2026-07-06T06:05:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.664, "last_started_at": "2026-07-06T06:03:15+09:00", "pid": 147886, "process": {"exists": false, "pid": 147886}, "restart_count": 884, "restart_count_per_hour": 19, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 885}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.139, "sample_count": 1240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1927787, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:05:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T06:05:06+09:00", "last_frame_read_age_sec": 0.259, "last_frame_read_at": "2026-07-06T06:05:06+09:00", "mjpeg": {"age_sec": 0.254, "exists": true, "mtime": "2026-07-06T06:05:06.564088+09:00", "mtime_epoch": 1783285506.564088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 150809}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:05:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.129, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.039, "pushed": 4.039, "read": 4.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.953, "count": 120, "max_ms": 29.198, "p95_ms": 28.044}, "gstreamer_push": {"avg_ms": 0.887, "count": 121, "max_ms": 0.97, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.1, "count": 120, "max_ms": 65.227, "p95_ms": 61.843}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.639, "count": 120, "max_ms": 91.041, "p95_ms": 87.729}, "yolo11_640_fire_11class": {"avg_ms": 49.725, "count": 120, "max_ms": 66.185, "p95_ms": 64.127}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.726, "count": 120, "max_ms": 88.805, "p95_ms": 84.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.777, "count": 121, "max_ms": 302.719, "p95_ms": 284.437}, "read_interval": {"avg_ms": 249.381, "count": 120, "max_ms": 313.728, "p95_ms": 294.373}, "triton_wait": {"avg_ms": 68.668, "count": 120, "max_ms": 91.053, "p95_ms": 87.742}, "visualize": {"avg_ms": 9.352, "count": 120, "max_ms": 11.698, "p95_ms": 11.237}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16064}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16400.805, "limit_mb": 8796093022207.996}, "detail": {"pid": 147993, "restart_count": 885, "start_count": 886}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 794, "h264_decode_error_count": 1942, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:05:06+09:00", "last_error_summary": "[h264 @ 0x55e757fc4500] error while decoding MB 54 9, bytestream -7", "last_exit_at": "2026-07-06T06:05:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:05:09+09:00", "pid": 147993, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.328, "name": "ffmpeg", "pid": 147993, "rss_mb": 139.328, "state": "S (sleeping)", "threads": 106}, "restart_count": 885, "restart_count_per_hour": 20, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 886}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.437, "sample_count": 1220, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1927787, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:05:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.419, "last_frame_pushed_at": "2026-07-06T06:05:06+09:00", "last_frame_read_age_sec": 4.67, "last_frame_read_at": "2026-07-06T06:05:06+09:00", "mjpeg": {"age_sec": 4.405, "exists": true, "mtime": "2026-07-06T06:05:06.824092+09:00", "mtime_epoch": 1783285506.8240917, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:05:11+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.129, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.402, "pushed": 3.402, "read": 3.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.072, "count": 101, "max_ms": 29.198, "p95_ms": 28.087}, "gstreamer_push": {"avg_ms": 0.889, "count": 102, "max_ms": 0.97, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.079, "count": 101, "max_ms": 64.595, "p95_ms": 61.825}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.519, "count": 101, "max_ms": 91.041, "p95_ms": 87.729}, "yolo11_640_fire_11class": {"avg_ms": 49.78, "count": 101, "max_ms": 66.185, "p95_ms": 64.377}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.994, "count": 101, "max_ms": 88.805, "p95_ms": 84.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.824, "count": 102, "max_ms": 302.719, "p95_ms": 285.015}, "read_interval": {"avg_ms": 252.452, "count": 101, "max_ms": 313.728, "p95_ms": 295.255}, "triton_wait": {"avg_ms": 68.613, "count": 101, "max_ms": 91.053, "p95_ms": 87.742}, "visualize": {"avg_ms": 9.558, "count": 101, "max_ms": 11.698, "p95_ms": 11.279}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16068}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16294.52, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560c56832b80] error while decoding MB 37 27, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 795, "h264_decode_error_count": 1943, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:07:01+09:00", "last_error_summary": "[h264 @ 0x560c56832b80] error while decoding MB 37 27, bytestream -14", "last_exit_at": "2026-07-06T06:05:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.202, "last_started_at": "2026-07-06T06:05:09+09:00", "pid": 147993, "process": {"exists": false, "pid": 147993}, "restart_count": 885, "restart_count_per_hour": 20, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 886}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 285.877, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1928206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:07:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T06:07:01+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-06T06:07:01+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T06:07:01.389609+09:00", "mtime_epoch": 1783285621.3896089, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148721}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:07:01+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.387, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.986, "pushed": 3.986, "read": 3.986}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.48, "count": 119, "max_ms": 28.934, "p95_ms": 27.715}, "gstreamer_push": {"avg_ms": 0.844, "count": 119, "max_ms": 0.941, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.667, "count": 119, "max_ms": 64.574, "p95_ms": 61.798}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.493, "count": 119, "max_ms": 98.863, "p95_ms": 90.962}, "yolo11_640_fire_11class": {"avg_ms": 50.508, "count": 119, "max_ms": 71.715, "p95_ms": 69.15}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.744, "count": 119, "max_ms": 90.619, "p95_ms": 87.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.452, "count": 119, "max_ms": 305.06, "p95_ms": 287.386}, "read_interval": {"avg_ms": 252.592, "count": 119, "max_ms": 314.677, "p95_ms": 296.328}, "triton_wait": {"avg_ms": 73.524, "count": 119, "max_ms": 98.875, "p95_ms": 90.974}, "visualize": {"avg_ms": 9.465, "count": 119, "max_ms": 12.857, "p95_ms": 11.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.858, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16068}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16294.094, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:07:01+09:00", "last_error_summary": "[h264 @ 0x560c56832b80] error while decoding MB 37 27, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 795, "h264_decode_error_count": 1943, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:07:01+09:00", "last_error_summary": "[h264 @ 0x560c56832b80] error while decoding MB 37 27, bytestream -14", "last_exit_at": "2026-07-06T06:07:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.401, "last_started_at": "2026-07-06T06:05:09+09:00", "pid": 147993, "process": {"exists": false, "pid": 147993}, "restart_count": 885, "restart_count_per_hour": 20, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 886}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 285.877, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1928207, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:07:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:07:01+09:00", "last_frame_read_age_sec": 0.249, "last_frame_read_at": "2026-07-06T06:07:01+09:00", "mjpeg": {"age_sec": 0.24, "exists": true, "mtime": "2026-07-06T06:07:01.389609+09:00", "mtime_epoch": 1783285621.3896089, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148721}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:07:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.387, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.993, "pushed": 3.993, "read": 3.959}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.477, "count": 119, "max_ms": 28.934, "p95_ms": 27.715}, "gstreamer_push": {"avg_ms": 0.844, "count": 119, "max_ms": 0.941, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.678, "count": 119, "max_ms": 64.574, "p95_ms": 61.798}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.454, "count": 119, "max_ms": 98.863, "p95_ms": 90.962}, "yolo11_640_fire_11class": {"avg_ms": 50.511, "count": 119, "max_ms": 71.715, "p95_ms": 69.15}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.759, "count": 119, "max_ms": 90.619, "p95_ms": 87.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.402, "count": 119, "max_ms": 305.06, "p95_ms": 287.386}, "read_interval": {"avg_ms": 252.53, "count": 118, "max_ms": 314.677, "p95_ms": 296.328}, "triton_wait": {"avg_ms": 73.486, "count": 119, "max_ms": 98.875, "p95_ms": 90.974}, "visualize": {"avg_ms": 9.465, "count": 119, "max_ms": 12.857, "p95_ms": 11.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.802, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16068}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16294.086, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 795, "h264_decode_error_count": 1943, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:07:01+09:00", "last_error_summary": "[h264 @ 0x560c56832b80] error while decoding MB 37 27, bytestream -14", "last_exit_at": "2026-07-06T06:07:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.405, "last_started_at": "2026-07-06T06:05:09+09:00", "pid": 147993, "process": {"exists": false, "pid": 147993}, "restart_count": 885, "restart_count_per_hour": 20, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 886}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 285.877, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1928207, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:07:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:07:01+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-06T06:07:01+09:00", "mjpeg": {"age_sec": 0.243, "exists": true, "mtime": "2026-07-06T06:07:01.389609+09:00", "mtime_epoch": 1783285621.3896089, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148721}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:07:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.387, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.993, "pushed": 3.993, "read": 3.959}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.477, "count": 119, "max_ms": 28.934, "p95_ms": 27.715}, "gstreamer_push": {"avg_ms": 0.844, "count": 119, "max_ms": 0.941, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.678, "count": 119, "max_ms": 64.574, "p95_ms": 61.798}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.454, "count": 119, "max_ms": 98.863, "p95_ms": 90.962}, "yolo11_640_fire_11class": {"avg_ms": 50.511, "count": 119, "max_ms": 71.715, "p95_ms": 69.15}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.759, "count": 119, "max_ms": 90.619, "p95_ms": 87.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.402, "count": 119, "max_ms": 305.06, "p95_ms": 287.386}, "read_interval": {"avg_ms": 252.53, "count": 118, "max_ms": 314.677, "p95_ms": 296.328}, "triton_wait": {"avg_ms": 73.486, "count": 119, "max_ms": 98.875, "p95_ms": 90.974}, "visualize": {"avg_ms": 9.465, "count": 119, "max_ms": 12.857, "p95_ms": 11.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.805, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16068}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16400.551, "limit_mb": 8796093022207.996}, "detail": {"pid": 148100, "restart_count": 886, "start_count": 887}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 795, "h264_decode_error_count": 1943, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:07:01+09:00", "last_error_summary": "[h264 @ 0x560c56832b80] error while decoding MB 37 27, bytestream -14", "last_exit_at": "2026-07-06T06:07:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:07:04+09:00", "pid": 148100, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.223, "name": "ffmpeg", "pid": 148100, "rss_mb": 139.223, "state": "S (sleeping)", "threads": 106}, "restart_count": 886, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 887}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 286.215, "sample_count": 1139, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1928207, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:07:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.575, "last_frame_pushed_at": "2026-07-06T06:07:01+09:00", "last_frame_read_age_sec": 4.821, "last_frame_read_at": "2026-07-06T06:07:01+09:00", "mjpeg": {"age_sec": 4.561, "exists": true, "mtime": "2026-07-06T06:07:01.641612+09:00", "mtime_epoch": 1783285621.641612, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144359}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:07:06+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.387, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.446, "pushed": 3.446, "read": 3.412}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.302, "count": 102, "max_ms": 28.934, "p95_ms": 27.557}, "gstreamer_push": {"avg_ms": 0.843, "count": 103, "max_ms": 0.92, "p95_ms": 0.876}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.956, "count": 102, "max_ms": 64.574, "p95_ms": 61.741}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.639, "count": 102, "max_ms": 98.863, "p95_ms": 90.962}, "yolo11_640_fire_11class": {"avg_ms": 50.873, "count": 102, "max_ms": 71.715, "p95_ms": 69.15}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.102, "count": 102, "max_ms": 90.619, "p95_ms": 87.402}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.299, "count": 103, "max_ms": 304.073, "p95_ms": 277.086}, "read_interval": {"avg_ms": 248.327, "count": 102, "max_ms": 314.677, "p95_ms": 287.112}, "triton_wait": {"avg_ms": 73.668, "count": 102, "max_ms": 98.875, "p95_ms": 90.974}, "visualize": {"avg_ms": 9.293, "count": 102, "max_ms": 12.857, "p95_ms": 11.273}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16071}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16295.527, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556236825880] cabac decode of qscale diff failed at 23 21 [h264 @ 0x556236825880] error while decoding MB 23 21, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 796, "h264_decode_error_count": 1945, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:09:12+09:00", "last_error_summary": "[h264 @ 0x556236825880] cabac decode of qscale diff failed at 23 21 [h264 @ 0x556236825880] error while decoding MB 23 21, bytestream -8", "last_exit_at": "2026-07-06T06:07:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.334, "last_started_at": "2026-07-06T06:07:04+09:00", "pid": 148100, "process": {"exists": false, "pid": 148100}, "restart_count": 886, "restart_count_per_hour": 20, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 887}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 390.336, "sample_count": 1005, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1928576, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:09:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T06:09:12+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T06:09:12+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-06T06:09:12.499349+09:00", "mtime_epoch": 1783285752.499349, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:09:12+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.645, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 74, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.488, "pushed": 2.488, "read": 2.488}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.875, "count": 74, "max_ms": 40.916, "p95_ms": 39.363}, "gstreamer_push": {"avg_ms": 0.866, "count": 74, "max_ms": 1.056, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.255, "count": 74, "max_ms": 64.784, "p95_ms": 61.936}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.385, "count": 74, "max_ms": 95.709, "p95_ms": 91.452}, "yolo11_640_fire_11class": {"avg_ms": 51.168, "count": 74, "max_ms": 71.008, "p95_ms": 66.841}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.852, "count": 74, "max_ms": 91.827, "p95_ms": 84.91}}, "truncated_count": 0}, "processing_total": {"avg_ms": 395.976, "count": 74, "max_ms": 496.861, "p95_ms": 456.614}, "read_interval": {"avg_ms": 405.642, "count": 74, "max_ms": 505.886, "p95_ms": 466.578}, "triton_wait": {"avg_ms": 72.302, "count": 74, "max_ms": 95.722, "p95_ms": 91.829}, "visualize": {"avg_ms": 12.605, "count": 74, "max_ms": 16.432, "p95_ms": 14.496}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.744, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16071}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16294.598, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:09:12+09:00", "last_error_summary": "[h264 @ 0x556236825880] cabac decode of qscale diff failed at 23 21 [h264 @ 0x556236825880] error while decoding MB 23 21, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 796, "h264_decode_error_count": 1945, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:09:12+09:00", "last_error_summary": "[h264 @ 0x556236825880] cabac decode of qscale diff failed at 23 21 [h264 @ 0x556236825880] error while decoding MB 23 21, bytestream -8", "last_exit_at": "2026-07-06T06:09:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.69, "last_started_at": "2026-07-06T06:07:04+09:00", "pid": 148100, "process": {"exists": false, "pid": 148100}, "restart_count": 886, "restart_count_per_hour": 20, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 887}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 390.817, "sample_count": 1004, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1928577, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:09:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:09:12+09:00", "last_frame_read_age_sec": 0.405, "last_frame_read_at": "2026-07-06T06:09:12+09:00", "mjpeg": {"age_sec": 0.392, "exists": true, "mtime": "2026-07-06T06:09:12.499349+09:00", "mtime_epoch": 1783285752.499349, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:09:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.645, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.488, "pushed": 2.488, "read": 2.455}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.918, "count": 74, "max_ms": 40.916, "p95_ms": 39.363}, "gstreamer_push": {"avg_ms": 0.864, "count": 74, "max_ms": 1.056, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.251, "count": 74, "max_ms": 64.784, "p95_ms": 61.936}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.256, "count": 74, "max_ms": 95.709, "p95_ms": 91.452}, "yolo11_640_fire_11class": {"avg_ms": 51.184, "count": 74, "max_ms": 71.008, "p95_ms": 66.841}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.786, "count": 74, "max_ms": 91.827, "p95_ms": 84.91}}, "truncated_count": 0}, "processing_total": {"avg_ms": 396.995, "count": 74, "max_ms": 496.861, "p95_ms": 456.614}, "read_interval": {"avg_ms": 406.627, "count": 73, "max_ms": 505.886, "p95_ms": 466.578}, "triton_wait": {"avg_ms": 72.173, "count": 74, "max_ms": 95.722, "p95_ms": 91.829}, "visualize": {"avg_ms": 12.625, "count": 74, "max_ms": 16.432, "p95_ms": 14.496}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.74, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16071}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16294.59, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 796, "h264_decode_error_count": 1945, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:09:12+09:00", "last_error_summary": "[h264 @ 0x556236825880] cabac decode of qscale diff failed at 23 21 [h264 @ 0x556236825880] error while decoding MB 23 21, bytestream -8", "last_exit_at": "2026-07-06T06:09:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.693, "last_started_at": "2026-07-06T06:07:04+09:00", "pid": 148100, "process": {"exists": false, "pid": 148100}, "restart_count": 886, "restart_count_per_hour": 20, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 887}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 390.817, "sample_count": 1004, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1928577, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:09:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T06:09:12+09:00", "last_frame_read_age_sec": 0.408, "last_frame_read_at": "2026-07-06T06:09:12+09:00", "mjpeg": {"age_sec": 0.395, "exists": true, "mtime": "2026-07-06T06:09:12.499349+09:00", "mtime_epoch": 1783285752.499349, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:09:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.645, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.488, "pushed": 2.488, "read": 2.454}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.918, "count": 74, "max_ms": 40.916, "p95_ms": 39.363}, "gstreamer_push": {"avg_ms": 0.864, "count": 74, "max_ms": 1.056, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.251, "count": 74, "max_ms": 64.784, "p95_ms": 61.936}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.256, "count": 74, "max_ms": 95.709, "p95_ms": 91.452}, "yolo11_640_fire_11class": {"avg_ms": 51.184, "count": 74, "max_ms": 71.008, "p95_ms": 66.841}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.786, "count": 74, "max_ms": 91.827, "p95_ms": 84.91}}, "truncated_count": 0}, "processing_total": {"avg_ms": 396.995, "count": 74, "max_ms": 496.861, "p95_ms": 456.614}, "read_interval": {"avg_ms": 406.627, "count": 73, "max_ms": 505.886, "p95_ms": 466.578}, "triton_wait": {"avg_ms": 72.173, "count": 74, "max_ms": 95.722, "p95_ms": 91.829}, "visualize": {"avg_ms": 12.625, "count": 74, "max_ms": 16.432, "p95_ms": 14.496}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.743, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16071}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16304.969, "limit_mb": 8796093022207.996}, "detail": {"pid": 148207, "restart_count": 887, "start_count": 888}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 796, "h264_decode_error_count": 1945, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:09:12+09:00", "last_error_summary": "[h264 @ 0x556236825880] cabac decode of qscale diff failed at 23 21 [h264 @ 0x556236825880] error while decoding MB 23 21, bytestream -8", "last_exit_at": "2026-07-06T06:09:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:09:15+09:00", "pid": 148207, "process": {"exists": true, "fd_count": 4, "hwm_mb": 43.789, "name": "ffmpeg", "pid": 148207, "rss_mb": 43.789, "state": "S (sleeping)", "threads": 1}, "restart_count": 887, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 888}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 391.013, "sample_count": 988, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1928577, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:09:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.35, "last_frame_pushed_at": "2026-07-06T06:09:12+09:00", "last_frame_read_age_sec": 4.75, "last_frame_read_at": "2026-07-06T06:09:12+09:00", "mjpeg": {"age_sec": 4.328, "exists": true, "mtime": "2026-07-06T06:09:12.907355+09:00", "mtime_epoch": 1783285752.9073546, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151182}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:09:17+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.645, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.103, "pushed": 2.103, "read": 2.07}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.399, "count": 62, "max_ms": 40.916, "p95_ms": 39.363}, "gstreamer_push": {"avg_ms": 0.864, "count": 63, "max_ms": 1.056, "p95_ms": 0.956}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.588, "count": 62, "max_ms": 64.784, "p95_ms": 61.555}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.152, "count": 62, "max_ms": 95.19, "p95_ms": 89.963}, "yolo11_640_fire_11class": {"avg_ms": 51.791, "count": 62, "max_ms": 71.008, "p95_ms": 66.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.268, "count": 62, "max_ms": 91.827, "p95_ms": 84.91}}, "truncated_count": 0}, "processing_total": {"avg_ms": 402.745, "count": 63, "max_ms": 496.861, "p95_ms": 456.614}, "read_interval": {"avg_ms": 412.464, "count": 62, "max_ms": 505.886, "p95_ms": 466.578}, "triton_wait": {"avg_ms": 72.243, "count": 62, "max_ms": 95.202, "p95_ms": 91.466}, "visualize": {"avg_ms": 12.707, "count": 62, "max_ms": 16.432, "p95_ms": 14.496}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16074}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16295.816, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559ef9cd6840] error while decoding MB 4 41, bytestream -33", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 797, "h264_decode_error_count": 1946, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:11:14+09:00", "last_error_summary": "[h264 @ 0x559ef9cd6840] error while decoding MB 4 41, bytestream -33", "last_exit_at": "2026-07-06T06:09:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.188, "last_started_at": "2026-07-06T06:09:15+09:00", "pid": 148207, "process": {"exists": false, "pid": 148207}, "restart_count": 887, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 888}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 391.013, "sample_count": 973, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1928999, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T06:11:14+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T06:11:14+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-06T06:11:14.368970+09:00", "mtime_epoch": 1783285874.36897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 143740}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:11:14+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.902, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.893, "pushed": 3.893, "read": 3.893}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.682, "count": 116, "max_ms": 33.195, "p95_ms": 30.725}, "gstreamer_push": {"avg_ms": 0.862, "count": 116, "max_ms": 1.139, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.281, "count": 116, "max_ms": 63.766, "p95_ms": 62.2}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.404, "count": 116, "max_ms": 94.147, "p95_ms": 85.63}, "yolo11_640_fire_11class": {"avg_ms": 51.361, "count": 116, "max_ms": 68.531, "p95_ms": 66.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.048, "count": 116, "max_ms": 95.543, "p95_ms": 90.315}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.79, "count": 116, "max_ms": 311.577, "p95_ms": 298.71}, "read_interval": {"avg_ms": 258.681, "count": 116, "max_ms": 321.709, "p95_ms": 308.042}, "triton_wait": {"avg_ms": 73.365, "count": 116, "max_ms": 95.555, "p95_ms": 90.479}, "visualize": {"avg_ms": 9.685, "count": 116, "max_ms": 12.704, "p95_ms": 12.416}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.794, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16075}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16295.336, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:11:14+09:00", "last_error_summary": "[h264 @ 0x559ef9cd6840] error while decoding MB 4 41, bytestream -33", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 797, "h264_decode_error_count": 1946, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:11:14+09:00", "last_error_summary": "[h264 @ 0x559ef9cd6840] error while decoding MB 4 41, bytestream -33", "last_exit_at": "2026-07-06T06:11:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.386, "last_started_at": "2026-07-06T06:09:15+09:00", "pid": 148207, "process": {"exists": false, "pid": 148207}, "restart_count": 887, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 888}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 391.013, "sample_count": 973, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929000, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:11:14+09:00", "last_frame_read_age_sec": 0.262, "last_frame_read_at": "2026-07-06T06:11:14+09:00", "mjpeg": {"age_sec": 0.252, "exists": true, "mtime": "2026-07-06T06:11:14.368970+09:00", "mtime_epoch": 1783285874.36897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 143740}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:11:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.902, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.901, "pushed": 3.901, "read": 3.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.715, "count": 116, "max_ms": 33.195, "p95_ms": 30.864}, "gstreamer_push": {"avg_ms": 0.862, "count": 117, "max_ms": 1.139, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.304, "count": 116, "max_ms": 63.766, "p95_ms": 62.2}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.447, "count": 116, "max_ms": 94.147, "p95_ms": 85.63}, "yolo11_640_fire_11class": {"avg_ms": 51.37, "count": 116, "max_ms": 68.531, "p95_ms": 66.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.035, "count": 116, "max_ms": 95.543, "p95_ms": 90.315}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.859, "count": 117, "max_ms": 311.577, "p95_ms": 298.71}, "read_interval": {"avg_ms": 258.681, "count": 116, "max_ms": 321.709, "p95_ms": 308.042}, "triton_wait": {"avg_ms": 73.389, "count": 116, "max_ms": 95.555, "p95_ms": 90.479}, "visualize": {"avg_ms": 9.688, "count": 116, "max_ms": 12.704, "p95_ms": 12.416}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16075}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16295.328, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 797, "h264_decode_error_count": 1946, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:11:14+09:00", "last_error_summary": "[h264 @ 0x559ef9cd6840] error while decoding MB 4 41, bytestream -33", "last_exit_at": "2026-07-06T06:11:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.389, "last_started_at": "2026-07-06T06:09:15+09:00", "pid": 148207, "process": {"exists": false, "pid": 148207}, "restart_count": 887, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 888}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 391.013, "sample_count": 973, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929000, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:11:14+09:00", "last_frame_read_age_sec": 0.265, "last_frame_read_at": "2026-07-06T06:11:14+09:00", "mjpeg": {"age_sec": 0.255, "exists": true, "mtime": "2026-07-06T06:11:14.368970+09:00", "mtime_epoch": 1783285874.36897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 143740}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:11:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.902, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.901, "pushed": 3.901, "read": 3.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.715, "count": 116, "max_ms": 33.195, "p95_ms": 30.864}, "gstreamer_push": {"avg_ms": 0.862, "count": 117, "max_ms": 1.139, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.304, "count": 116, "max_ms": 63.766, "p95_ms": 62.2}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.447, "count": 116, "max_ms": 94.147, "p95_ms": 85.63}, "yolo11_640_fire_11class": {"avg_ms": 51.37, "count": 116, "max_ms": 68.531, "p95_ms": 66.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.035, "count": 116, "max_ms": 95.543, "p95_ms": 90.315}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.859, "count": 117, "max_ms": 311.577, "p95_ms": 298.71}, "read_interval": {"avg_ms": 258.681, "count": 116, "max_ms": 321.709, "p95_ms": 308.042}, "triton_wait": {"avg_ms": 73.389, "count": 116, "max_ms": 95.555, "p95_ms": 90.479}, "visualize": {"avg_ms": 9.688, "count": 116, "max_ms": 12.704, "p95_ms": 12.416}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 337, "tracking_summary": 16075}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16404.18, "limit_mb": 8796093022207.996}, "detail": {"pid": 148314, "restart_count": 888, "start_count": 889}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 797, "h264_decode_error_count": 1946, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:11:14+09:00", "last_error_summary": "[h264 @ 0x559ef9cd6840] error while decoding MB 4 41, bytestream -33", "last_exit_at": "2026-07-06T06:11:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:11:17+09:00", "pid": 148314, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.457, "name": "ffmpeg", "pid": 148314, "rss_mb": 139.457, "state": "S (sleeping)", "threads": 106}, "restart_count": 888, "restart_count_per_hour": 22, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 889}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 391.904, "sample_count": 956, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929000, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.839, "last_frame_pushed_at": "2026-07-06T06:11:14+09:00", "last_frame_read_age_sec": 5.098, "last_frame_read_at": "2026-07-06T06:11:14+09:00", "mjpeg": {"age_sec": 4.824, "exists": true, "mtime": "2026-07-06T06:11:14.632973+09:00", "mtime_epoch": 1783285874.6329734, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 143316}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:11:19+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9545.902, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.346, "pushed": 3.346, "read": 3.313}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.659, "count": 99, "max_ms": 33.195, "p95_ms": 30.906}, "gstreamer_push": {"avg_ms": 0.863, "count": 100, "max_ms": 1.139, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.452, "count": 99, "max_ms": 63.766, "p95_ms": 62.157}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.021, "count": 99, "max_ms": 94.147, "p95_ms": 87.829}, "yolo11_640_fire_11class": {"avg_ms": 51.385, "count": 99, "max_ms": 68.531, "p95_ms": 66.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.988, "count": 99, "max_ms": 95.543, "p95_ms": 90.315}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.715, "count": 100, "max_ms": 308.212, "p95_ms": 293.771}, "read_interval": {"avg_ms": 253.333, "count": 99, "max_ms": 317.361, "p95_ms": 304.394}, "triton_wait": {"avg_ms": 73.854, "count": 99, "max_ms": 95.555, "p95_ms": 90.479}, "visualize": {"avg_ms": 9.424, "count": 99, "max_ms": 12.554, "p95_ms": 12.136}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 338, "tracking_summary": 16078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16301.527, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55806838e740] error while decoding MB 26 35, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 798, "h264_decode_error_count": 1947, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:13:05+09:00", "last_error_summary": "[h264 @ 0x55806838e740] error while decoding MB 26 35, bytestream -11", "last_exit_at": "2026-07-06T06:11:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.278, "last_started_at": "2026-07-06T06:11:17+09:00", "pid": 148314, "process": {"exists": false, "pid": 148314}, "restart_count": 888, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 889}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 388.861, "sample_count": 1036, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929432, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:13:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T06:13:05+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-06T06:13:05+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-06T06:13:05.698453+09:00", "mtime_epoch": 1783285985.6984527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:13:05+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.16, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.151, "pushed": 4.151, "read": 4.151}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.951, "count": 124, "max_ms": 31.745, "p95_ms": 30.668}, "gstreamer_push": {"avg_ms": 0.893, "count": 124, "max_ms": 1.154, "p95_ms": 1.073}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.174, "count": 124, "max_ms": 71.926, "p95_ms": 65.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.425, "count": 124, "max_ms": 94.465, "p95_ms": 85.895}, "yolo11_640_fire_11class": {"avg_ms": 48.726, "count": 124, "max_ms": 70.812, "p95_ms": 65.554}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.07, "count": 124, "max_ms": 92.228, "p95_ms": 86.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.328, "count": 124, "max_ms": 926.708, "p95_ms": 316.417}, "read_interval": {"avg_ms": 242.063, "count": 124, "max_ms": 937.362, "p95_ms": 326.544}, "triton_wait": {"avg_ms": 70.386, "count": 124, "max_ms": 94.477, "p95_ms": 87.535}, "visualize": {"avg_ms": 9.119, "count": 124, "max_ms": 19.036, "p95_ms": 14.92}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 338, "tracking_summary": 16078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16301.008, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:13:05+09:00", "last_error_summary": "[h264 @ 0x55806838e740] error while decoding MB 26 35, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 798, "h264_decode_error_count": 1947, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:13:05+09:00", "last_error_summary": "[h264 @ 0x55806838e740] error while decoding MB 26 35, bytestream -11", "last_exit_at": "2026-07-06T06:13:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.412, "last_started_at": "2026-07-06T06:11:17+09:00", "pid": 148314, "process": {"exists": false, "pid": 148314}, "restart_count": 888, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 889}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 388.861, "sample_count": 1036, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929433, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:13:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:13:05+09:00", "last_frame_read_age_sec": 0.182, "last_frame_read_at": "2026-07-06T06:13:05+09:00", "mjpeg": {"age_sec": 0.17, "exists": true, "mtime": "2026-07-06T06:13:05.698453+09:00", "mtime_epoch": 1783285985.6984527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:13:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.134, "pushed": 4.134, "read": 4.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.944, "count": 124, "max_ms": 31.745, "p95_ms": 30.668}, "gstreamer_push": {"avg_ms": 0.894, "count": 124, "max_ms": 1.154, "p95_ms": 1.073}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.265, "count": 124, "max_ms": 71.926, "p95_ms": 65.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.422, "count": 124, "max_ms": 94.465, "p95_ms": 85.895}, "yolo11_640_fire_11class": {"avg_ms": 48.781, "count": 124, "max_ms": 70.812, "p95_ms": 65.554}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.15, "count": 124, "max_ms": 92.228, "p95_ms": 86.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.213, "count": 124, "max_ms": 926.708, "p95_ms": 316.417}, "read_interval": {"avg_ms": 242.063, "count": 124, "max_ms": 937.362, "p95_ms": 326.544}, "triton_wait": {"avg_ms": 70.384, "count": 124, "max_ms": 94.477, "p95_ms": 87.535}, "visualize": {"avg_ms": 9.114, "count": 124, "max_ms": 19.036, "p95_ms": 14.92}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 338, "tracking_summary": 16078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16301.031, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 798, "h264_decode_error_count": 1947, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:13:05+09:00", "last_error_summary": "[h264 @ 0x55806838e740] error while decoding MB 26 35, bytestream -11", "last_exit_at": "2026-07-06T06:13:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.416, "last_started_at": "2026-07-06T06:11:17+09:00", "pid": 148314, "process": {"exists": false, "pid": 148314}, "restart_count": 888, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 889}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 388.861, "sample_count": 1036, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929433, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:13:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:13:05+09:00", "last_frame_read_age_sec": 0.185, "last_frame_read_at": "2026-07-06T06:13:05+09:00", "mjpeg": {"age_sec": 0.173, "exists": true, "mtime": "2026-07-06T06:13:05.698453+09:00", "mtime_epoch": 1783285985.6984527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:13:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.159, "pushed": 4.159, "read": 4.125}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.944, "count": 124, "max_ms": 31.745, "p95_ms": 30.668}, "gstreamer_push": {"avg_ms": 0.894, "count": 124, "max_ms": 1.154, "p95_ms": 1.073}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.265, "count": 124, "max_ms": 71.926, "p95_ms": 65.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.422, "count": 124, "max_ms": 94.465, "p95_ms": 85.895}, "yolo11_640_fire_11class": {"avg_ms": 48.781, "count": 124, "max_ms": 70.812, "p95_ms": 65.554}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.15, "count": 124, "max_ms": 92.228, "p95_ms": 86.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.213, "count": 124, "max_ms": 926.708, "p95_ms": 316.417}, "read_interval": {"avg_ms": 242.403, "count": 123, "max_ms": 937.362, "p95_ms": 326.544}, "triton_wait": {"avg_ms": 70.384, "count": 124, "max_ms": 94.477, "p95_ms": 87.535}, "visualize": {"avg_ms": 9.114, "count": 124, "max_ms": 19.036, "p95_ms": 14.92}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.817, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 338, "tracking_summary": 16078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16417.266, "limit_mb": 8796093022207.996}, "detail": {"pid": 148481, "restart_count": 889, "start_count": 890}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 798, "h264_decode_error_count": 1947, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:13:05+09:00", "last_error_summary": "[h264 @ 0x55806838e740] error while decoding MB 26 35, bytestream -11", "last_exit_at": "2026-07-06T06:13:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:13:08+09:00", "pid": 148481, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.559, "name": "ffmpeg", "pid": 148481, "rss_mb": 147.559, "state": "S (sleeping)", "threads": 106}, "restart_count": 889, "restart_count_per_hour": 22, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 890}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 389.187, "sample_count": 1021, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929433, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:13:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.663, "last_frame_pushed_at": "2026-07-06T06:13:05+09:00", "last_frame_read_age_sec": 4.842, "last_frame_read_at": "2026-07-06T06:13:05+09:00", "mjpeg": {"age_sec": 4.646, "exists": true, "mtime": "2026-07-06T06:13:05.882455+09:00", "mtime_epoch": 1783285985.882455, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157526}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:13:10+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.16, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.472, "pushed": 3.472, "read": 3.439}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.094, "count": 103, "max_ms": 31.745, "p95_ms": 30.684}, "gstreamer_push": {"avg_ms": 0.901, "count": 104, "max_ms": 1.154, "p95_ms": 1.076}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.63, "count": 103, "max_ms": 71.926, "p95_ms": 65.07}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.264, "count": 103, "max_ms": 94.465, "p95_ms": 86.22}, "yolo11_640_fire_11class": {"avg_ms": 49.21, "count": 103, "max_ms": 67.705, "p95_ms": 63.993}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.777, "count": 103, "max_ms": 92.228, "p95_ms": 86.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.881, "count": 104, "max_ms": 926.708, "p95_ms": 317.799}, "read_interval": {"avg_ms": 246.227, "count": 103, "max_ms": 937.362, "p95_ms": 328.603}, "triton_wait": {"avg_ms": 70.231, "count": 103, "max_ms": 94.477, "p95_ms": 88.178}, "visualize": {"avg_ms": 9.283, "count": 103, "max_ms": 19.036, "p95_ms": 15.25}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 338, "tracking_summary": 16082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16302.359, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5636fb50ba80] error while decoding MB 59 29, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 799, "h264_decode_error_count": 1948, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:15:08+09:00", "last_error_summary": "[h264 @ 0x5636fb50ba80] error while decoding MB 59 29, bytestream -5", "last_exit_at": "2026-07-06T06:13:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.185, "last_started_at": "2026-07-06T06:13:08+09:00", "pid": 148481, "process": {"exists": false, "pid": 148481}, "restart_count": 889, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 890}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 319.674, "sample_count": 1100, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:15:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-06T06:15:08+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-06T06:15:08+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-06T06:15:08.660092+09:00", "mtime_epoch": 1783286108.660092, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:15:08+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.16, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.569, "pushed": 3.569, "read": 3.569}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.014, "count": 106, "max_ms": 31.844, "p95_ms": 30.668}, "gstreamer_push": {"avg_ms": 0.866, "count": 107, "max_ms": 1.115, "p95_ms": 0.939}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.763, "count": 106, "max_ms": 65.735, "p95_ms": 61.793}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.958, "count": 106, "max_ms": 90.351, "p95_ms": 80.076}, "yolo11_640_fire_11class": {"avg_ms": 47.996, "count": 106, "max_ms": 65.695, "p95_ms": 63.434}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.762, "count": 106, "max_ms": 92.214, "p95_ms": 89.334}}, "truncated_count": 0}, "processing_total": {"avg_ms": 272.414, "count": 107, "max_ms": 383.159, "p95_ms": 320.882}, "read_interval": {"avg_ms": 282.202, "count": 107, "max_ms": 395.627, "p95_ms": 330.036}, "triton_wait": {"avg_ms": 70.234, "count": 106, "max_ms": 92.227, "p95_ms": 90.296}, "visualize": {"avg_ms": 10.828, "count": 106, "max_ms": 14.455, "p95_ms": 13.331}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 338, "tracking_summary": 16082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16301.777, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:15:08+09:00", "last_error_summary": "[h264 @ 0x5636fb50ba80] error while decoding MB 59 29, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 799, "h264_decode_error_count": 1948, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:15:08+09:00", "last_error_summary": "[h264 @ 0x5636fb50ba80] error while decoding MB 59 29, bytestream -5", "last_exit_at": "2026-07-06T06:15:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.411, "last_started_at": "2026-07-06T06:13:08+09:00", "pid": 148481, "process": {"exists": false, "pid": 148481}, "restart_count": 889, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 890}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 319.674, "sample_count": 1100, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929860, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:15:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:15:08+09:00", "last_frame_read_age_sec": 0.284, "last_frame_read_at": "2026-07-06T06:15:08+09:00", "mjpeg": {"age_sec": 0.278, "exists": true, "mtime": "2026-07-06T06:15:08.660092+09:00", "mtime_epoch": 1783286108.660092, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:15:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.58, "pushed": 3.58, "read": 3.546}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.051, "count": 106, "max_ms": 31.844, "p95_ms": 30.894}, "gstreamer_push": {"avg_ms": 0.866, "count": 107, "max_ms": 1.115, "p95_ms": 0.939}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.628, "count": 106, "max_ms": 65.735, "p95_ms": 60.51}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.747, "count": 106, "max_ms": 90.351, "p95_ms": 77.419}, "yolo11_640_fire_11class": {"avg_ms": 47.821, "count": 106, "max_ms": 65.695, "p95_ms": 63.434}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.546, "count": 106, "max_ms": 92.214, "p95_ms": 89.334}}, "truncated_count": 0}, "processing_total": {"avg_ms": 272.447, "count": 107, "max_ms": 383.159, "p95_ms": 320.882}, "read_interval": {"avg_ms": 282.151, "count": 106, "max_ms": 395.627, "p95_ms": 330.036}, "triton_wait": {"avg_ms": 70.022, "count": 106, "max_ms": 92.227, "p95_ms": 90.275}, "visualize": {"avg_ms": 10.823, "count": 106, "max_ms": 14.455, "p95_ms": 13.331}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 338, "tracking_summary": 16082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16301.77, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 799, "h264_decode_error_count": 1948, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:15:08+09:00", "last_error_summary": "[h264 @ 0x5636fb50ba80] error while decoding MB 59 29, bytestream -5", "last_exit_at": "2026-07-06T06:15:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.415, "last_started_at": "2026-07-06T06:13:08+09:00", "pid": 148481, "process": {"exists": false, "pid": 148481}, "restart_count": 889, "restart_count_per_hour": 21, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 890}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 319.674, "sample_count": 1100, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929860, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:15:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:15:08+09:00", "last_frame_read_age_sec": 0.288, "last_frame_read_at": "2026-07-06T06:15:08+09:00", "mjpeg": {"age_sec": 0.282, "exists": true, "mtime": "2026-07-06T06:15:08.660092+09:00", "mtime_epoch": 1783286108.660092, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:15:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.579, "pushed": 3.579, "read": 3.546}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.051, "count": 106, "max_ms": 31.844, "p95_ms": 30.894}, "gstreamer_push": {"avg_ms": 0.866, "count": 107, "max_ms": 1.115, "p95_ms": 0.939}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.628, "count": 106, "max_ms": 65.735, "p95_ms": 60.51}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.747, "count": 106, "max_ms": 90.351, "p95_ms": 77.419}, "yolo11_640_fire_11class": {"avg_ms": 47.821, "count": 106, "max_ms": 65.695, "p95_ms": 63.434}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.546, "count": 106, "max_ms": 92.214, "p95_ms": 89.334}}, "truncated_count": 0}, "processing_total": {"avg_ms": 272.447, "count": 107, "max_ms": 383.159, "p95_ms": 320.882}, "read_interval": {"avg_ms": 282.151, "count": 106, "max_ms": 395.627, "p95_ms": 330.036}, "triton_wait": {"avg_ms": 70.022, "count": 106, "max_ms": 92.227, "p95_ms": 90.275}, "visualize": {"avg_ms": 10.823, "count": 106, "max_ms": 14.455, "p95_ms": 13.331}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 338, "tracking_summary": 16082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16409.77, "limit_mb": 8796093022207.996}, "detail": {"pid": 148588, "restart_count": 890, "start_count": 891}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 799, "h264_decode_error_count": 1948, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:15:08+09:00", "last_error_summary": "[h264 @ 0x5636fb50ba80] error while decoding MB 59 29, bytestream -5", "last_exit_at": "2026-07-06T06:15:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:15:11+09:00", "pid": 148588, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.172, "name": "ffmpeg", "pid": 148588, "rss_mb": 139.172, "state": "S (sleeping)", "threads": 106}, "restart_count": 890, "restart_count_per_hour": 22, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 891}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 319.737, "sample_count": 1085, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1929860, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:15:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.369, "last_frame_pushed_at": "2026-07-06T06:15:08+09:00", "last_frame_read_age_sec": 4.65, "last_frame_read_at": "2026-07-06T06:15:08+09:00", "mjpeg": {"age_sec": 4.353, "exists": true, "mtime": "2026-07-06T06:15:08.952096+09:00", "mtime_epoch": 1783286108.9520962, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168617}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:15:13+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.16, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.022, "pushed": 3.022, "read": 2.988}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.178, "count": 90, "max_ms": 31.844, "p95_ms": 31.134}, "gstreamer_push": {"avg_ms": 0.866, "count": 90, "max_ms": 1.108, "p95_ms": 0.939}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.912, "count": 90, "max_ms": 65.735, "p95_ms": 61.793}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.269, "count": 90, "max_ms": 90.351, "p95_ms": 80.076}, "yolo11_640_fire_11class": {"avg_ms": 48.126, "count": 90, "max_ms": 65.695, "p95_ms": 63.479}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.947, "count": 90, "max_ms": 92.214, "p95_ms": 90.264}}, "truncated_count": 0}, "processing_total": {"avg_ms": 275.488, "count": 90, "max_ms": 383.159, "p95_ms": 321.092}, "read_interval": {"avg_ms": 285.209, "count": 89, "max_ms": 395.627, "p95_ms": 330.16}, "triton_wait": {"avg_ms": 71.125, "count": 90, "max_ms": 92.227, "p95_ms": 90.302}, "visualize": {"avg_ms": 10.934, "count": 90, "max_ms": 14.455, "p95_ms": 13.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.784, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 344, "tracking_summary": 16085}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16313.785, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 800, "h264_decode_error_count": 1949, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:16+09:00", "last_error_summary": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "last_exit_at": "2026-07-06T06:15:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.213, "last_started_at": "2026-07-06T06:15:11+09:00", "pid": 148588, "process": {"exists": false, "pid": 148588}, "restart_count": 890, "restart_count_per_hour": 22, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 891}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 398.215, "sample_count": 990, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930218, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:17:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T06:17:16+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-06T06:17:16+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T06:17:16.477798+09:00", "mtime_epoch": 1783286236.477798, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:17:16+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.418, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 86, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.884, "pushed": 2.884, "read": 2.884}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.02, "count": 85, "max_ms": 36.132, "p95_ms": 33.997}, "gstreamer_push": {"avg_ms": 0.853, "count": 86, "max_ms": 1.062, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.962, "count": 85, "max_ms": 63.176, "p95_ms": 60.561}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.606, "count": 85, "max_ms": 96.892, "p95_ms": 90.37}, "yolo11_640_fire_11class": {"avg_ms": 47.331, "count": 85, "max_ms": 62.787, "p95_ms": 61.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.442, "count": 85, "max_ms": 94.939, "p95_ms": 91.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 340.333, "count": 86, "max_ms": 401.687, "p95_ms": 390.329}, "read_interval": {"avg_ms": 349.732, "count": 86, "max_ms": 411.709, "p95_ms": 399.866}, "triton_wait": {"avg_ms": 70.539, "count": 85, "max_ms": 96.904, "p95_ms": 94.023}, "visualize": {"avg_ms": 12.991, "count": 85, "max_ms": 17.138, "p95_ms": 14.918}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.824, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 344, "tracking_summary": 16085}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16313.156, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:17:16+09:00", "last_error_summary": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 800, "h264_decode_error_count": 1949, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:16+09:00", "last_error_summary": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "last_exit_at": "2026-07-06T06:17:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.53, "last_started_at": "2026-07-06T06:15:11+09:00", "pid": 148588, "process": {"exists": false, "pid": 148588}, "restart_count": 890, "restart_count_per_hour": 22, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 891}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 398.215, "sample_count": 989, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930219, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:17:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:17:16+09:00", "last_frame_read_age_sec": 0.366, "last_frame_read_at": "2026-07-06T06:17:16+09:00", "mjpeg": {"age_sec": 0.356, "exists": true, "mtime": "2026-07-06T06:17:16.477798+09:00", "mtime_epoch": 1783286236.477798, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:17:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.418, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.885, "pushed": 2.885, "read": 2.851}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.99, "count": 85, "max_ms": 36.132, "p95_ms": 33.997}, "gstreamer_push": {"avg_ms": 0.853, "count": 86, "max_ms": 1.062, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.947, "count": 85, "max_ms": 63.176, "p95_ms": 60.561}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.758, "count": 85, "max_ms": 96.892, "p95_ms": 90.37}, "yolo11_640_fire_11class": {"avg_ms": 47.317, "count": 85, "max_ms": 62.787, "p95_ms": 61.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.497, "count": 85, "max_ms": 94.939, "p95_ms": 91.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 341.027, "count": 86, "max_ms": 401.687, "p95_ms": 390.329}, "read_interval": {"avg_ms": 350.18, "count": 85, "max_ms": 411.709, "p95_ms": 399.866}, "triton_wait": {"avg_ms": 70.639, "count": 85, "max_ms": 96.904, "p95_ms": 94.023}, "visualize": {"avg_ms": 13.009, "count": 85, "max_ms": 17.138, "p95_ms": 14.918}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.812, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 344, "tracking_summary": 16085}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16313.148, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 800, "h264_decode_error_count": 1949, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:16+09:00", "last_error_summary": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "last_exit_at": "2026-07-06T06:17:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.534, "last_started_at": "2026-07-06T06:15:11+09:00", "pid": 148588, "process": {"exists": false, "pid": 148588}, "restart_count": 890, "restart_count_per_hour": 22, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 891}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 398.215, "sample_count": 989, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930219, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:17:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:17:16+09:00", "last_frame_read_age_sec": 0.369, "last_frame_read_at": "2026-07-06T06:17:16+09:00", "mjpeg": {"age_sec": 0.359, "exists": true, "mtime": "2026-07-06T06:17:16.477798+09:00", "mtime_epoch": 1783286236.477798, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:17:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.418, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.884, "pushed": 2.884, "read": 2.851}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.99, "count": 85, "max_ms": 36.132, "p95_ms": 33.997}, "gstreamer_push": {"avg_ms": 0.853, "count": 86, "max_ms": 1.062, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.947, "count": 85, "max_ms": 63.176, "p95_ms": 60.561}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.758, "count": 85, "max_ms": 96.892, "p95_ms": 90.37}, "yolo11_640_fire_11class": {"avg_ms": 47.317, "count": 85, "max_ms": 62.787, "p95_ms": 61.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.497, "count": 85, "max_ms": 94.939, "p95_ms": 91.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 341.027, "count": 86, "max_ms": 401.687, "p95_ms": 390.329}, "read_interval": {"avg_ms": 350.18, "count": 85, "max_ms": 411.709, "p95_ms": 399.866}, "triton_wait": {"avg_ms": 70.639, "count": 85, "max_ms": 96.904, "p95_ms": 94.023}, "visualize": {"avg_ms": 13.009, "count": 85, "max_ms": 17.138, "p95_ms": 14.918}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.815, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 344, "tracking_summary": 16085}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16430.562, "limit_mb": 8796093022207.996}, "detail": {"pid": 148755, "restart_count": 891, "start_count": 892}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 800, "h264_decode_error_count": 1949, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:16+09:00", "last_error_summary": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "last_exit_at": "2026-07-06T06:17:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:17:19+09:00", "pid": 148755, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.211, "name": "ffmpeg", "pid": 148755, "rss_mb": 147.211, "state": "S (sleeping)", "threads": 106}, "restart_count": 891, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 892}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 398.215, "sample_count": 972, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930219, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:17:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.624, "last_frame_pushed_at": "2026-07-06T06:17:16+09:00", "last_frame_read_age_sec": 4.988, "last_frame_read_at": "2026-07-06T06:17:16+09:00", "mjpeg": {"age_sec": 4.61, "exists": true, "mtime": "2026-07-06T06:17:16.845803+09:00", "mtime_epoch": 1783286236.845803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176683}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:17:21+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.418, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.383, "pushed": 2.383, "read": 2.349}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.376, "count": 71, "max_ms": 36.132, "p95_ms": 34.793}, "gstreamer_push": {"avg_ms": 0.85, "count": 71, "max_ms": 1.062, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.695, "count": 71, "max_ms": 63.176, "p95_ms": 60.652}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.542, "count": 71, "max_ms": 96.892, "p95_ms": 92.408}, "yolo11_640_fire_11class": {"avg_ms": 47.199, "count": 71, "max_ms": 62.787, "p95_ms": 62.285}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.826, "count": 71, "max_ms": 94.939, "p95_ms": 91.476}}, "truncated_count": 0}, "processing_total": {"avg_ms": 349.737, "count": 71, "max_ms": 401.687, "p95_ms": 392.693}, "read_interval": {"avg_ms": 358.909, "count": 70, "max_ms": 411.709, "p95_ms": 401.538}, "triton_wait": {"avg_ms": 70.317, "count": 71, "max_ms": 96.904, "p95_ms": 94.036}, "visualize": {"avg_ms": 13.248, "count": 71, "max_ms": 17.138, "p95_ms": 14.918}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.798, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20038}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16162.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d5ad83db80] error while decoding MB 17 28, bytestream -7 [in#0/rtsp @ 0x55d5ad007e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5ad007e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 141, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:30+09:00", "last_error_summary": "[h264 @ 0x55d5ad83db80] error while decoding MB 17 28, bytestream -7 [in#0/rtsp @ 0x55d5ad007e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5ad007e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-05T16:11:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 3056.871, "last_started_at": "2026-07-06T05:26:33+09:00", "pid": 146090, "process": {"exists": false, "pid": 146090}, "restart_count": 142, "restart_count_per_hour": 1, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.858, "sample_count": 2463, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2404604, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:17:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-06T06:17:30+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-06T06:17:30+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-06T06:17:30.585986+09:00", "mtime_epoch": 1783286250.5859864, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 151745}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:17:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2588.309, "name": "celery", "pid": 267, "rss_mb": 2588.309, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 55, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.837, "pushed": 1.837, "read": 1.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.293, "count": 56, "max_ms": 12.034, "p95_ms": 11.931}, "gstreamer_push": {"avg_ms": 1.118, "count": 55, "max_ms": 1.395, "p95_ms": 1.34}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.798, "count": 56, "max_ms": 54.597, "p95_ms": 52.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.754, "count": 56, "max_ms": 89.111, "p95_ms": 75.419}, "yolo11_640_fire_11class": {"avg_ms": 46.561, "count": 56, "max_ms": 60.24, "p95_ms": 54.136}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.333, "count": 56, "max_ms": 85.232, "p95_ms": 76.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.383, "count": 55, "max_ms": 142.573, "p95_ms": 129.838}, "read_interval": {"avg_ms": 544.842, "count": 55, "max_ms": 14094.124, "p95_ms": 156.151}, "triton_wait": {"avg_ms": 53.952, "count": 56, "max_ms": 89.124, "p95_ms": 85.245}, "visualize": {"avg_ms": 4.366, "count": 56, "max_ms": 5.588, "p95_ms": 5.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20038}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16165.195, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:17:30+09:00", "last_error_summary": "[h264 @ 0x55d5ad83db80] error while decoding MB 17 28, bytestream -7 [in#0/rtsp @ 0x55d5ad007e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5ad007e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 141, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:30+09:00", "last_error_summary": "[h264 @ 0x55d5ad83db80] error while decoding MB 17 28, bytestream -7 [in#0/rtsp @ 0x55d5ad007e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5ad007e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:17:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 3056.891, "last_started_at": "2026-07-06T05:26:33+09:00", "pid": 146090, "process": {"exists": false, "pid": 146090}, "restart_count": 142, "restart_count_per_hour": 1, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.858, "sample_count": 2464, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2404604, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:17:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T06:17:30+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-06T06:17:30+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-06T06:17:30.585986+09:00", "mtime_epoch": 1783286250.5859864, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 151745}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:17:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2588.309, "name": "celery", "pid": 267, "rss_mb": 2588.309, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.869, "pushed": 1.869, "read": 1.836}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.293, "count": 56, "max_ms": 12.034, "p95_ms": 11.931}, "gstreamer_push": {"avg_ms": 1.124, "count": 56, "max_ms": 1.42, "p95_ms": 1.367}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.798, "count": 56, "max_ms": 54.597, "p95_ms": 52.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.754, "count": 56, "max_ms": 89.111, "p95_ms": 75.419}, "yolo11_640_fire_11class": {"avg_ms": 46.561, "count": 56, "max_ms": 60.24, "p95_ms": 54.136}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.333, "count": 56, "max_ms": 85.232, "p95_ms": 76.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.032, "count": 56, "max_ms": 142.573, "p95_ms": 129.838}, "read_interval": {"avg_ms": 544.842, "count": 55, "max_ms": 14094.124, "p95_ms": 156.151}, "triton_wait": {"avg_ms": 53.952, "count": 56, "max_ms": 89.124, "p95_ms": 85.245}, "visualize": {"avg_ms": 4.366, "count": 56, "max_ms": 5.588, "p95_ms": 5.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20038}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16165.141, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 141, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:30+09:00", "last_error_summary": "[h264 @ 0x55d5ad83db80] error while decoding MB 17 28, bytestream -7 [in#0/rtsp @ 0x55d5ad007e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5ad007e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:17:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 3056.899, "last_started_at": "2026-07-06T05:26:33+09:00", "pid": 146090, "process": {"exists": false, "pid": 146090}, "restart_count": 142, "restart_count_per_hour": 1, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.858, "sample_count": 2464, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2404604, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:17:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-06T06:17:30+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-06T06:17:30+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-06T06:17:30.585986+09:00", "mtime_epoch": 1783286250.5859864, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 151745}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:17:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2588.309, "name": "celery", "pid": 267, "rss_mb": 2588.309, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.869, "pushed": 1.869, "read": 1.836}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.293, "count": 56, "max_ms": 12.034, "p95_ms": 11.931}, "gstreamer_push": {"avg_ms": 1.124, "count": 56, "max_ms": 1.42, "p95_ms": 1.367}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.798, "count": 56, "max_ms": 54.597, "p95_ms": 52.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.754, "count": 56, "max_ms": 89.111, "p95_ms": 75.419}, "yolo11_640_fire_11class": {"avg_ms": 46.561, "count": 56, "max_ms": 60.24, "p95_ms": 54.136}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.333, "count": 56, "max_ms": 85.232, "p95_ms": 76.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.032, "count": 56, "max_ms": 142.573, "p95_ms": 129.838}, "read_interval": {"avg_ms": 544.842, "count": 55, "max_ms": 14094.124, "p95_ms": 156.151}, "triton_wait": {"avg_ms": 53.952, "count": 56, "max_ms": 89.124, "p95_ms": 85.245}, "visualize": {"avg_ms": 4.366, "count": 56, "max_ms": 5.588, "p95_ms": 5.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20038}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16210.531, "limit_mb": 8796093022207.996}, "detail": {"pid": 148862, "restart_count": 143, "start_count": 144}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 141, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:30+09:00", "last_error_summary": "[h264 @ 0x55d5ad83db80] error while decoding MB 17 28, bytestream -7 [in#0/rtsp @ 0x55d5ad007e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5ad007e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:17:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:17:33+09:00", "pid": 148862, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.68, "name": "ffmpeg", "pid": 148862, "rss_mb": 57.68, "state": "S (sleeping)", "threads": 1}, "restart_count": 143, "restart_count_per_hour": 2, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.24, "sample_count": 2427, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2404604, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:17:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.415, "last_frame_pushed_at": "2026-07-06T06:17:30+09:00", "last_frame_read_age_sec": 4.49, "last_frame_read_at": "2026-07-06T06:17:30+09:00", "mjpeg": {"age_sec": 4.394, "exists": true, "mtime": "2026-07-06T06:17:30.673988+09:00", "mtime_epoch": 1783286250.6739876, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 151338}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:17:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2588.309, "name": "celery", "pid": 267, "rss_mb": 2588.309, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 16, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.533, "pushed": 0.533, "read": 0.533}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.84, "count": 16, "max_ms": 12.003, "p95_ms": 12.003}, "gstreamer_push": {"avg_ms": 1.194, "count": 16, "max_ms": 1.42, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.034, "count": 16, "max_ms": 53.493, "p95_ms": 53.493}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.893, "count": 16, "max_ms": 87.846, "p95_ms": 87.846}, "yolo11_640_fire_11class": {"avg_ms": 46.397, "count": 16, "max_ms": 54.136, "p95_ms": 54.136}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.38, "count": 16, "max_ms": 76.302, "p95_ms": 76.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.718, "count": 16, "max_ms": 106.88, "p95_ms": 106.88}, "read_interval": {"avg_ms": 1599.26, "count": 16, "max_ms": 14094.124, "p95_ms": 14094.124}, "triton_wait": {"avg_ms": 53.004, "count": 16, "max_ms": 87.858, "p95_ms": 87.858}, "visualize": {"avg_ms": 4.033, "count": 16, "max_ms": 5.588, "p95_ms": 5.588}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 354, "tracking_summary": 16087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16327.98, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:17:16+09:00", "last_error_summary": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 800, "h264_decode_error_count": 1949, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:16+09:00", "last_error_summary": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "last_exit_at": "2026-07-06T06:19:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.373, "last_started_at": "2026-07-06T06:17:19+09:00", "pid": 148755, "process": {"exists": false, "pid": 148755}, "restart_count": 891, "restart_count_per_hour": 22, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 892}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 510.703, "sample_count": 757, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:19:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:19:36+09:00", "last_frame_read_age_sec": 0.475, "last_frame_read_at": "2026-07-06T06:19:36+09:00", "mjpeg": {"age_sec": 0.464, "exists": true, "mtime": "2026-07-06T06:19:36.363667+09:00", "mtime_epoch": 1783286376.3636665, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185711}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:19:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.676, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.166, "pushed": 2.166, "read": 2.132}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 38.213, "count": 64, "max_ms": 44.618, "p95_ms": 42.1}, "gstreamer_push": {"avg_ms": 0.866, "count": 64, "max_ms": 0.942, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.652, "count": 64, "max_ms": 72.552, "p95_ms": 68.707}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.648, "count": 64, "max_ms": 93.614, "p95_ms": 86.919}, "yolo11_640_fire_11class": {"avg_ms": 48.724, "count": 64, "max_ms": 66.546, "p95_ms": 64.295}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.888, "count": 64, "max_ms": 91.686, "p95_ms": 86.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 459.498, "count": 64, "max_ms": 648.163, "p95_ms": 558.095}, "read_interval": {"avg_ms": 468.868, "count": 63, "max_ms": 658.011, "p95_ms": 568.518}, "triton_wait": {"avg_ms": 73.957, "count": 64, "max_ms": 93.626, "p95_ms": 89.102}, "visualize": {"avg_ms": 22.077, "count": 64, "max_ms": 27.726, "p95_ms": 26.295}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.547, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 354, "tracking_summary": 16087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16327.973, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 800, "h264_decode_error_count": 1949, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:16+09:00", "last_error_summary": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "last_exit_at": "2026-07-06T06:19:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.376, "last_started_at": "2026-07-06T06:17:19+09:00", "pid": 148755, "process": {"exists": false, "pid": 148755}, "restart_count": 891, "restart_count_per_hour": 22, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 892}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 510.703, "sample_count": 757, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:19:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:19:36+09:00", "last_frame_read_age_sec": 0.478, "last_frame_read_at": "2026-07-06T06:19:36+09:00", "mjpeg": {"age_sec": 0.467, "exists": true, "mtime": "2026-07-06T06:19:36.363667+09:00", "mtime_epoch": 1783286376.3636665, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185711}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:19:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.676, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.166, "pushed": 2.166, "read": 2.132}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 38.213, "count": 64, "max_ms": 44.618, "p95_ms": 42.1}, "gstreamer_push": {"avg_ms": 0.866, "count": 64, "max_ms": 0.942, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.652, "count": 64, "max_ms": 72.552, "p95_ms": 68.707}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.648, "count": 64, "max_ms": 93.614, "p95_ms": 86.919}, "yolo11_640_fire_11class": {"avg_ms": 48.724, "count": 64, "max_ms": 66.546, "p95_ms": 64.295}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.888, "count": 64, "max_ms": 91.686, "p95_ms": 86.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 459.498, "count": 64, "max_ms": 648.163, "p95_ms": 558.095}, "read_interval": {"avg_ms": 468.868, "count": 63, "max_ms": 658.011, "p95_ms": 568.518}, "triton_wait": {"avg_ms": 73.957, "count": 64, "max_ms": 93.626, "p95_ms": 89.102}, "visualize": {"avg_ms": 22.077, "count": 64, "max_ms": 27.726, "p95_ms": 26.295}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.55, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 354, "tracking_summary": 16087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16435.844, "limit_mb": 8796093022207.996}, "detail": {"pid": 149074, "restart_count": 892, "start_count": 893}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 800, "h264_decode_error_count": 1949, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:17:16+09:00", "last_error_summary": "[h264 @ 0x55c139e6a540] error while decoding MB 73 8, bytestream -6", "last_exit_at": "2026-07-06T06:19:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:19:39+09:00", "pid": 149074, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.191, "name": "ffmpeg", "pid": 149074, "rss_mb": 139.191, "state": "S (sleeping)", "threads": 106}, "restart_count": 892, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 893}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 512.638, "sample_count": 743, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:19:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.361, "last_frame_pushed_at": "2026-07-06T06:19:36+09:00", "last_frame_read_age_sec": 4.834, "last_frame_read_at": "2026-07-06T06:19:36+09:00", "mjpeg": {"age_sec": 4.342, "exists": true, "mtime": "2026-07-06T06:19:36.843673+09:00", "mtime_epoch": 1783286376.8436728, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185459}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:19:41+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.676, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.876, "pushed": 1.876, "read": 1.842}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 38.2, "count": 55, "max_ms": 44.618, "p95_ms": 43.214}, "gstreamer_push": {"avg_ms": 0.864, "count": 56, "max_ms": 0.942, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.974, "count": 55, "max_ms": 72.552, "p95_ms": 70.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.693, "count": 55, "max_ms": 93.614, "p95_ms": 88.445}, "yolo11_640_fire_11class": {"avg_ms": 48.547, "count": 55, "max_ms": 66.546, "p95_ms": 64.359}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.22, "count": 55, "max_ms": 91.686, "p95_ms": 89.09}}, "truncated_count": 0}, "processing_total": {"avg_ms": 457.419, "count": 56, "max_ms": 648.163, "p95_ms": 562.304}, "read_interval": {"avg_ms": 466.723, "count": 55, "max_ms": 658.011, "p95_ms": 573.33}, "triton_wait": {"avg_ms": 73.879, "count": 55, "max_ms": 93.626, "p95_ms": 90.516}, "visualize": {"avg_ms": 21.781, "count": 55, "max_ms": 27.726, "p95_ms": 26.295}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20054}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16271.281, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562f864a58c0] error while decoding MB 97 40, bytestream -33 [in#0/rtsp @ 0x562f86472e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f86472e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 142, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:21:30+09:00", "last_error_summary": "[h264 @ 0x562f864a58c0] error while decoding MB 97 40, bytestream -33 [in#0/rtsp @ 0x562f86472e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f86472e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:17:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 237.902, "last_started_at": "2026-07-06T06:17:33+09:00", "pid": 148862, "process": {"exists": false, "pid": 148862}, "restart_count": 143, "restart_count_per_hour": 2, "rtsp_timeout_count": 244, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.294, "sample_count": 2324, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2406534, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:21:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T06:21:30+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T06:21:30+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-06T06:21:30.909197+09:00", "mtime_epoch": 1783286490.9091973, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 178294}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:21:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2589.34, "name": "celery", "pid": 267, "rss_mb": 2589.34, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 32, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.101, "pushed": 1.101, "read": 1.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.291, "count": 32, "max_ms": 14.27, "p95_ms": 12.713}, "gstreamer_push": {"avg_ms": 1.094, "count": 33, "max_ms": 1.286, "p95_ms": 1.268}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.618, "count": 32, "max_ms": 55.84, "p95_ms": 51.652}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.226, "count": 32, "max_ms": 88.386, "p95_ms": 74.076}, "yolo11_640_fire_11class": {"avg_ms": 50.709, "count": 32, "max_ms": 61.294, "p95_ms": 54.076}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.097, "count": 32, "max_ms": 86.981, "p95_ms": 74.065}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.424, "count": 33, "max_ms": 128.492, "p95_ms": 119.117}, "read_interval": {"avg_ms": 937.981, "count": 32, "max_ms": 15995.437, "p95_ms": 10743.849}, "triton_wait": {"avg_ms": 53.21, "count": 32, "max_ms": 88.395, "p95_ms": 74.086}, "visualize": {"avg_ms": 4.554, "count": 32, "max_ms": 5.698, "p95_ms": 5.601}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20054}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16271.133, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:21:30+09:00", "last_error_summary": "[h264 @ 0x562f864a58c0] error while decoding MB 97 40, bytestream -33 [in#0/rtsp @ 0x562f86472e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f86472e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 142, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:21:30+09:00", "last_error_summary": "[h264 @ 0x562f864a58c0] error while decoding MB 97 40, bytestream -33 [in#0/rtsp @ 0x562f86472e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f86472e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:21:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 237.912, "last_started_at": "2026-07-06T06:17:33+09:00", "pid": 148862, "process": {"exists": false, "pid": 148862}, "restart_count": 143, "restart_count_per_hour": 2, "rtsp_timeout_count": 244, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.294, "sample_count": 2324, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2406534, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:21:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T06:21:30+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-06T06:21:30+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-06T06:21:30.909197+09:00", "mtime_epoch": 1783286490.9091973, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 178294}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:21:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2589.34, "name": "celery", "pid": 267, "rss_mb": 2589.34, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 32, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.1, "pushed": 1.1, "read": 1.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.291, "count": 32, "max_ms": 14.27, "p95_ms": 12.713}, "gstreamer_push": {"avg_ms": 1.094, "count": 33, "max_ms": 1.286, "p95_ms": 1.268}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.618, "count": 32, "max_ms": 55.84, "p95_ms": 51.652}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.226, "count": 32, "max_ms": 88.386, "p95_ms": 74.076}, "yolo11_640_fire_11class": {"avg_ms": 50.709, "count": 32, "max_ms": 61.294, "p95_ms": 54.076}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.097, "count": 32, "max_ms": 86.981, "p95_ms": 74.065}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.424, "count": 33, "max_ms": 128.492, "p95_ms": 119.117}, "read_interval": {"avg_ms": 937.981, "count": 32, "max_ms": 15995.437, "p95_ms": 10743.849}, "triton_wait": {"avg_ms": 53.21, "count": 32, "max_ms": 88.395, "p95_ms": 74.086}, "visualize": {"avg_ms": 4.554, "count": 32, "max_ms": 5.698, "p95_ms": 5.601}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20054}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16271.051, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 142, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:21:30+09:00", "last_error_summary": "[h264 @ 0x562f864a58c0] error while decoding MB 97 40, bytestream -33 [in#0/rtsp @ 0x562f86472e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f86472e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:21:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 237.92, "last_started_at": "2026-07-06T06:17:33+09:00", "pid": 148862, "process": {"exists": false, "pid": 148862}, "restart_count": 143, "restart_count_per_hour": 2, "rtsp_timeout_count": 244, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.294, "sample_count": 2324, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2406534, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:21:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.022, "last_frame_pushed_at": "2026-07-06T06:21:30+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-06T06:21:30+09:00", "mjpeg": {"age_sec": 0.006, "exists": true, "mtime": "2026-07-06T06:21:30.981198+09:00", "mtime_epoch": 1783286490.981198, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 177518}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:21:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2589.34, "name": "celery", "pid": 267, "rss_mb": 2589.34, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 32, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.1, "pushed": 1.1, "read": 1.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.291, "count": 32, "max_ms": 14.27, "p95_ms": 12.713}, "gstreamer_push": {"avg_ms": 1.094, "count": 33, "max_ms": 1.286, "p95_ms": 1.268}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.618, "count": 32, "max_ms": 55.84, "p95_ms": 51.652}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.226, "count": 32, "max_ms": 88.386, "p95_ms": 74.076}, "yolo11_640_fire_11class": {"avg_ms": 50.709, "count": 32, "max_ms": 61.294, "p95_ms": 54.076}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.097, "count": 32, "max_ms": 86.981, "p95_ms": 74.065}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.424, "count": 33, "max_ms": 128.492, "p95_ms": 119.117}, "read_interval": {"avg_ms": 937.981, "count": 32, "max_ms": 15995.437, "p95_ms": 10743.849}, "triton_wait": {"avg_ms": 53.21, "count": 32, "max_ms": 88.395, "p95_ms": 74.086}, "visualize": {"avg_ms": 4.554, "count": 32, "max_ms": 5.698, "p95_ms": 5.601}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20054}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16415.555, "limit_mb": 8796093022207.996}, "detail": {"pid": 149241, "restart_count": 144, "start_count": 145}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 142, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:21:30+09:00", "last_error_summary": "[h264 @ 0x562f864a58c0] error while decoding MB 97 40, bytestream -33 [in#0/rtsp @ 0x562f86472e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f86472e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:21:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:21:33+09:00", "pid": 149241, "process": {"exists": true, "fd_count": 5, "hwm_mb": 172.816, "name": "ffmpeg", "pid": 149241, "rss_mb": 172.816, "state": "S (sleeping)", "threads": 106}, "restart_count": 144, "restart_count_per_hour": 3, "rtsp_timeout_count": 244, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.294, "sample_count": 2275, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2406534, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:21:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.613, "last_frame_pushed_at": "2026-07-06T06:21:30+09:00", "last_frame_read_age_sec": 4.675, "last_frame_read_at": "2026-07-06T06:21:30+09:00", "mjpeg": {"age_sec": 4.596, "exists": true, "mtime": "2026-07-06T06:21:30.981198+09:00", "mtime_epoch": 1783286490.981198, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 177518}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:21:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2589.34, "name": "celery", "pid": 267, "rss_mb": 2589.34, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.6, "pushed": 0.6, "read": 0.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.003, "count": 13, "max_ms": 11.422, "p95_ms": 11.422}, "gstreamer_push": {"avg_ms": 1.169, "count": 13, "max_ms": 1.286, "p95_ms": 1.286}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.621, "count": 13, "max_ms": 55.84, "p95_ms": 55.84}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.681, "count": 13, "max_ms": 88.386, "p95_ms": 88.386}, "yolo11_640_fire_11class": {"avg_ms": 51.631, "count": 13, "max_ms": 61.294, "p95_ms": 61.294}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.609, "count": 13, "max_ms": 86.981, "p95_ms": 86.981}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.078, "count": 13, "max_ms": 106.759, "p95_ms": 106.759}, "read_interval": {"avg_ms": 2133.45, "count": 13, "max_ms": 15995.437, "p95_ms": 15995.437}, "triton_wait": {"avg_ms": 53.866, "count": 13, "max_ms": 88.395, "p95_ms": 88.395}, "visualize": {"avg_ms": 3.858, "count": 13, "max_ms": 5.049, "p95_ms": 5.049}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.666, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 357, "tracking_summary": 16089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16337.449, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560b0c12d040] error while decoding MB 3 40, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 801, "h264_decode_error_count": 1950, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:21:51+09:00", "last_error_summary": "[h264 @ 0x560b0c12d040] error while decoding MB 3 40, bytestream -7", "last_exit_at": "2026-07-06T06:19:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.068, "last_started_at": "2026-07-06T06:19:39+09:00", "pid": 149074, "process": {"exists": false, "pid": 149074}, "restart_count": 892, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 893}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 534.447, "sample_count": 638, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930785, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:21:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T06:21:51+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T06:21:51+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T06:21:51.213469+09:00", "mtime_epoch": 1783286511.2134686, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183481}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:21:51+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.676, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 72, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.421, "pushed": 2.421, "read": 2.421}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.574, "count": 71, "max_ms": 44.338, "p95_ms": 42.106}, "gstreamer_push": {"avg_ms": 0.856, "count": 72, "max_ms": 0.914, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.698, "count": 71, "max_ms": 62.178, "p95_ms": 60.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.627, "count": 71, "max_ms": 90.868, "p95_ms": 85.288}, "yolo11_640_fire_11class": {"avg_ms": 49.042, "count": 71, "max_ms": 67.382, "p95_ms": 66.669}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.085, "count": 71, "max_ms": 96.037, "p95_ms": 91.857}}, "truncated_count": 0}, "processing_total": {"avg_ms": 410.25, "count": 72, "max_ms": 525.911, "p95_ms": 513.348}, "read_interval": {"avg_ms": 419.611, "count": 72, "max_ms": 534.786, "p95_ms": 522.237}, "triton_wait": {"avg_ms": 73.955, "count": 71, "max_ms": 96.049, "p95_ms": 91.869}, "visualize": {"avg_ms": 14.576, "count": 71, "max_ms": 21.631, "p95_ms": 20.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.736, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 357, "tracking_summary": 16089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16336.457, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:21:51+09:00", "last_error_summary": "[h264 @ 0x560b0c12d040] error while decoding MB 3 40, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 801, "h264_decode_error_count": 1950, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:21:51+09:00", "last_error_summary": "[h264 @ 0x560b0c12d040] error while decoding MB 3 40, bytestream -7", "last_exit_at": "2026-07-06T06:21:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.479, "last_started_at": "2026-07-06T06:19:39+09:00", "pid": 149074, "process": {"exists": false, "pid": 149074}, "restart_count": 892, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 893}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 534.447, "sample_count": 637, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930786, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:21:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:21:51+09:00", "last_frame_read_age_sec": 0.46, "last_frame_read_at": "2026-07-06T06:21:51+09:00", "mjpeg": {"age_sec": 0.45, "exists": true, "mtime": "2026-07-06T06:21:51.213469+09:00", "mtime_epoch": 1783286511.2134686, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183481}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:21:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.676, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.43, "pushed": 2.43, "read": 2.396}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.644, "count": 71, "max_ms": 45.127, "p95_ms": 42.461}, "gstreamer_push": {"avg_ms": 0.856, "count": 72, "max_ms": 0.914, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.894, "count": 71, "max_ms": 62.178, "p95_ms": 60.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.709, "count": 71, "max_ms": 90.868, "p95_ms": 85.288}, "yolo11_640_fire_11class": {"avg_ms": 49.163, "count": 71, "max_ms": 67.382, "p95_ms": 66.669}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.313, "count": 71, "max_ms": 96.037, "p95_ms": 91.857}}, "truncated_count": 0}, "processing_total": {"avg_ms": 409.27, "count": 72, "max_ms": 520.425, "p95_ms": 510.616}, "read_interval": {"avg_ms": 417.989, "count": 71, "max_ms": 530.604, "p95_ms": 519.602}, "triton_wait": {"avg_ms": 74.079, "count": 71, "max_ms": 96.049, "p95_ms": 91.869}, "visualize": {"avg_ms": 14.49, "count": 71, "max_ms": 21.631, "p95_ms": 20.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.628, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 357, "tracking_summary": 16089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16336.449, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 801, "h264_decode_error_count": 1950, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:21:51+09:00", "last_error_summary": "[h264 @ 0x560b0c12d040] error while decoding MB 3 40, bytestream -7", "last_exit_at": "2026-07-06T06:21:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.482, "last_started_at": "2026-07-06T06:19:39+09:00", "pid": 149074, "process": {"exists": false, "pid": 149074}, "restart_count": 892, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 893}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 534.447, "sample_count": 637, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930786, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:21:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:21:51+09:00", "last_frame_read_age_sec": 0.463, "last_frame_read_at": "2026-07-06T06:21:51+09:00", "mjpeg": {"age_sec": 0.453, "exists": true, "mtime": "2026-07-06T06:21:51.213469+09:00", "mtime_epoch": 1783286511.2134686, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183481}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:21:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.676, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.43, "pushed": 2.43, "read": 2.396}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.644, "count": 71, "max_ms": 45.127, "p95_ms": 42.461}, "gstreamer_push": {"avg_ms": 0.856, "count": 72, "max_ms": 0.914, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.894, "count": 71, "max_ms": 62.178, "p95_ms": 60.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.709, "count": 71, "max_ms": 90.868, "p95_ms": 85.288}, "yolo11_640_fire_11class": {"avg_ms": 49.163, "count": 71, "max_ms": 67.382, "p95_ms": 66.669}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.313, "count": 71, "max_ms": 96.037, "p95_ms": 91.857}}, "truncated_count": 0}, "processing_total": {"avg_ms": 409.27, "count": 72, "max_ms": 520.425, "p95_ms": 510.616}, "read_interval": {"avg_ms": 417.989, "count": 71, "max_ms": 530.604, "p95_ms": 519.602}, "triton_wait": {"avg_ms": 74.079, "count": 71, "max_ms": 96.049, "p95_ms": 91.869}, "visualize": {"avg_ms": 14.49, "count": 71, "max_ms": 21.631, "p95_ms": 20.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.63, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 357, "tracking_summary": 16089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16447.203, "limit_mb": 8796093022207.996}, "detail": {"pid": 149393, "restart_count": 893, "start_count": 894}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 801, "h264_decode_error_count": 1950, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:21:51+09:00", "last_error_summary": "[h264 @ 0x560b0c12d040] error while decoding MB 3 40, bytestream -7", "last_exit_at": "2026-07-06T06:21:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:21:53+09:00", "pid": 149393, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.125, "name": "ffmpeg", "pid": 149393, "rss_mb": 141.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 893, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 894}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 534.465, "sample_count": 625, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930786, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:21:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.231, "last_frame_pushed_at": "2026-07-06T06:21:51+09:00", "last_frame_read_age_sec": 4.688, "last_frame_read_at": "2026-07-06T06:21:51+09:00", "mjpeg": {"age_sec": 4.214, "exists": true, "mtime": "2026-07-06T06:21:51.677475+09:00", "mtime_epoch": 1783286511.6774747, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184752}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:21:55+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.676, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.149, "pushed": 2.149, "read": 2.115}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.033, "count": 63, "max_ms": 45.127, "p95_ms": 40.478}, "gstreamer_push": {"avg_ms": 0.857, "count": 64, "max_ms": 0.914, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.352, "count": 63, "max_ms": 62.178, "p95_ms": 60.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.823, "count": 63, "max_ms": 90.868, "p95_ms": 85.288}, "yolo11_640_fire_11class": {"avg_ms": 49.691, "count": 63, "max_ms": 67.382, "p95_ms": 66.669}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.111, "count": 63, "max_ms": 96.037, "p95_ms": 91.857}}, "truncated_count": 0}, "processing_total": {"avg_ms": 397.501, "count": 64, "max_ms": 489.209, "p95_ms": 472.59}, "read_interval": {"avg_ms": 405.941, "count": 63, "max_ms": 499.692, "p95_ms": 481.192}, "triton_wait": {"avg_ms": 74.366, "count": 63, "max_ms": 96.049, "p95_ms": 91.869}, "visualize": {"avg_ms": 13.744, "count": 63, "max_ms": 20.922, "p95_ms": 18.264}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.785, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 369, "tracking_summary": 16091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16358.828, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5629c63a70c0] error while decoding MB 59 38, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 802, "h264_decode_error_count": 1951, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:24:28+09:00", "last_error_summary": "[h264 @ 0x5629c63a70c0] error while decoding MB 59 38, bytestream -9", "last_exit_at": "2026-07-06T06:21:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 154.956, "last_started_at": "2026-07-06T06:21:53+09:00", "pid": 149393, "process": {"exists": false, "pid": 149393}, "restart_count": 893, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 894}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 765.519, "sample_count": 512, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930998, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:24:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T06:24:28+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T06:24:28+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T06:24:28.807576+09:00", "mtime_epoch": 1783286668.807576, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:24:28+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 41, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.4, "pushed": 1.4, "read": 1.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 57.894, "count": 41, "max_ms": 63.399, "p95_ms": 61.304}, "gstreamer_push": {"avg_ms": 0.887, "count": 41, "max_ms": 1.259, "p95_ms": 0.973}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.286, "count": 41, "max_ms": 70.045, "p95_ms": 67.787}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.766, "count": 41, "max_ms": 94.521, "p95_ms": 89.521}, "yolo11_640_fire_11class": {"avg_ms": 49.302, "count": 41, "max_ms": 69.225, "p95_ms": 64.574}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.488, "count": 41, "max_ms": 94.263, "p95_ms": 86.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 721.159, "count": 41, "max_ms": 898.5, "p95_ms": 786.682}, "read_interval": {"avg_ms": 730.587, "count": 41, "max_ms": 907.224, "p95_ms": 795.378}, "triton_wait": {"avg_ms": 76.268, "count": 41, "max_ms": 94.533, "p95_ms": 90.681}, "visualize": {"avg_ms": 22.532, "count": 41, "max_ms": 25.135, "p95_ms": 24.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.287, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 369, "tracking_summary": 16091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16357.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:24:28+09:00", "last_error_summary": "[h264 @ 0x5629c63a70c0] error while decoding MB 59 38, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 802, "h264_decode_error_count": 1951, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:24:28+09:00", "last_error_summary": "[h264 @ 0x5629c63a70c0] error while decoding MB 59 38, bytestream -9", "last_exit_at": "2026-07-06T06:24:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.661, "last_started_at": "2026-07-06T06:21:53+09:00", "pid": 149393, "process": {"exists": false, "pid": 149393}, "restart_count": 893, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 894}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 765.519, "sample_count": 511, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930999, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:24:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:24:29+09:00", "last_frame_read_age_sec": 0.755, "last_frame_read_at": "2026-07-06T06:24:28+09:00", "mjpeg": {"age_sec": 0.744, "exists": true, "mtime": "2026-07-06T06:24:28.807576+09:00", "mtime_epoch": 1783286668.807576, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:24:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 41, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.4, "pushed": 1.4, "read": 1.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 57.79, "count": 41, "max_ms": 63.399, "p95_ms": 61.164}, "gstreamer_push": {"avg_ms": 0.887, "count": 42, "max_ms": 1.259, "p95_ms": 0.973}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.597, "count": 41, "max_ms": 70.045, "p95_ms": 67.787}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.933, "count": 41, "max_ms": 94.521, "p95_ms": 89.521}, "yolo11_640_fire_11class": {"avg_ms": 49.791, "count": 41, "max_ms": 69.225, "p95_ms": 67.003}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.94, "count": 41, "max_ms": 94.263, "p95_ms": 86.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 721.849, "count": 42, "max_ms": 898.5, "p95_ms": 786.682}, "read_interval": {"avg_ms": 730.587, "count": 41, "max_ms": 907.224, "p95_ms": 795.378}, "triton_wait": {"avg_ms": 76.477, "count": 41, "max_ms": 94.533, "p95_ms": 90.681}, "visualize": {"avg_ms": 22.618, "count": 41, "max_ms": 27.758, "p95_ms": 25.022}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 369, "tracking_summary": 16091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16357.758, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 802, "h264_decode_error_count": 1951, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:24:28+09:00", "last_error_summary": "[h264 @ 0x5629c63a70c0] error while decoding MB 59 38, bytestream -9", "last_exit_at": "2026-07-06T06:24:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.664, "last_started_at": "2026-07-06T06:21:53+09:00", "pid": 149393, "process": {"exists": false, "pid": 149393}, "restart_count": 893, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 894}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 765.519, "sample_count": 511, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930999, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:24:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:24:29+09:00", "last_frame_read_age_sec": 0.758, "last_frame_read_at": "2026-07-06T06:24:28+09:00", "mjpeg": {"age_sec": 0.747, "exists": true, "mtime": "2026-07-06T06:24:28.807576+09:00", "mtime_epoch": 1783286668.807576, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:24:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 41, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.4, "pushed": 1.4, "read": 1.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 57.79, "count": 41, "max_ms": 63.399, "p95_ms": 61.164}, "gstreamer_push": {"avg_ms": 0.887, "count": 42, "max_ms": 1.259, "p95_ms": 0.973}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.597, "count": 41, "max_ms": 70.045, "p95_ms": 67.787}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.933, "count": 41, "max_ms": 94.521, "p95_ms": 89.521}, "yolo11_640_fire_11class": {"avg_ms": 49.791, "count": 41, "max_ms": 69.225, "p95_ms": 67.003}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.94, "count": 41, "max_ms": 94.263, "p95_ms": 86.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 721.849, "count": 42, "max_ms": 898.5, "p95_ms": 786.682}, "read_interval": {"avg_ms": 730.587, "count": 41, "max_ms": 907.224, "p95_ms": 795.378}, "triton_wait": {"avg_ms": 76.477, "count": 41, "max_ms": 94.533, "p95_ms": 90.681}, "visualize": {"avg_ms": 22.618, "count": 41, "max_ms": 27.758, "p95_ms": 25.022}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 369, "tracking_summary": 16091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16466.715, "limit_mb": 8796093022207.996}, "detail": {"pid": 149560, "restart_count": 894, "start_count": 895}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 802, "h264_decode_error_count": 1951, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:24:28+09:00", "last_error_summary": "[h264 @ 0x5629c63a70c0] error while decoding MB 59 38, bytestream -9", "last_exit_at": "2026-07-06T06:24:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:24:31+09:00", "pid": 149560, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.246, "name": "ffmpeg", "pid": 149560, "rss_mb": 139.246, "state": "S (sleeping)", "threads": 106}, "restart_count": 894, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 895}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 765.864, "sample_count": 502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1930999, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:24:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.305, "last_frame_pushed_at": "2026-07-06T06:24:29+09:00", "last_frame_read_age_sec": 5.057, "last_frame_read_at": "2026-07-06T06:24:28+09:00", "mjpeg": {"age_sec": 4.286, "exists": true, "mtime": "2026-07-06T06:24:29.567586+09:00", "mtime_epoch": 1783286669.567586, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202850}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:24:33+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 35, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.204, "pushed": 1.204, "read": 1.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 57.667, "count": 35, "max_ms": 63.399, "p95_ms": 61.304}, "gstreamer_push": {"avg_ms": 0.888, "count": 36, "max_ms": 1.259, "p95_ms": 1.249}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.572, "count": 35, "max_ms": 70.045, "p95_ms": 69.024}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.976, "count": 35, "max_ms": 94.521, "p95_ms": 90.669}, "yolo11_640_fire_11class": {"avg_ms": 50.121, "count": 35, "max_ms": 69.225, "p95_ms": 67.421}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.108, "count": 35, "max_ms": 94.263, "p95_ms": 88.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 721.863, "count": 36, "max_ms": 898.5, "p95_ms": 833.664}, "read_interval": {"avg_ms": 730.467, "count": 35, "max_ms": 907.224, "p95_ms": 843.326}, "triton_wait": {"avg_ms": 76.612, "count": 35, "max_ms": 94.533, "p95_ms": 94.275}, "visualize": {"avg_ms": 22.498, "count": 35, "max_ms": 27.758, "p95_ms": 25.135}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20069}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16296.543, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c42a5ed000] error while decoding MB 56 39, bytestream -5 [in#0/rtsp @ 0x55c42a5a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c42a5a5e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:25:26+09:00", "last_error_summary": "[h264 @ 0x55c42a5ed000] error while decoding MB 56 39, bytestream -5 [in#0/rtsp @ 0x55c42a5a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c42a5a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:21:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.265, "last_started_at": "2026-07-06T06:21:33+09:00", "pid": 149241, "process": {"exists": false, "pid": 149241}, "restart_count": 144, "restart_count_per_hour": 3, "rtsp_timeout_count": 246, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.799, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2408384, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:25:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T06:25:26+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T06:25:26+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-06T06:25:26.784351+09:00", "mtime_epoch": 1783286726.7843513, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 179372}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:25:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2590.371, "name": "celery", "pid": 267, "rss_mb": 2590.371, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 16, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.534, "pushed": 0.534, "read": 0.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.302, "count": 17, "max_ms": 9.316, "p95_ms": 9.316}, "gstreamer_push": {"avg_ms": 1.116, "count": 16, "max_ms": 1.215, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.354, "count": 17, "max_ms": 51.825, "p95_ms": 51.825}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.951, "count": 17, "max_ms": 53.13, "p95_ms": 53.13}, "yolo11_640_fire_11class": {"avg_ms": 49.335, "count": 17, "max_ms": 51.835, "p95_ms": 51.835}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.477, "count": 17, "max_ms": 50.809, "p95_ms": 50.809}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.482, "count": 16, "max_ms": 100.049, "p95_ms": 100.049}, "read_interval": {"avg_ms": 1872.045, "count": 16, "max_ms": 16335.901, "p95_ms": 16335.901}, "triton_wait": {"avg_ms": 50.163, "count": 17, "max_ms": 53.138, "p95_ms": 53.138}, "visualize": {"avg_ms": 3.578, "count": 17, "max_ms": 4.544, "p95_ms": 4.544}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20069}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16296.258, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:25:26+09:00", "last_error_summary": "[h264 @ 0x55c42a5ed000] error while decoding MB 56 39, bytestream -5 [in#0/rtsp @ 0x55c42a5a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c42a5a5e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:25:26+09:00", "last_error_summary": "[h264 @ 0x55c42a5ed000] error while decoding MB 56 39, bytestream -5 [in#0/rtsp @ 0x55c42a5a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c42a5a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:25:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.279, "last_started_at": "2026-07-06T06:21:33+09:00", "pid": 149241, "process": {"exists": false, "pid": 149241}, "restart_count": 144, "restart_count_per_hour": 3, "rtsp_timeout_count": 246, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.799, "sample_count": 2228, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2408384, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:25:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T06:25:26+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-06T06:25:26+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-06T06:25:26.784351+09:00", "mtime_epoch": 1783286726.7843513, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 179372}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:25:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2590.371, "name": "celery", "pid": 267, "rss_mb": 2590.371, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 17, "pushed_frames": 17, "read_frames": 16, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.567, "pushed": 0.567, "read": 0.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.302, "count": 17, "max_ms": 9.316, "p95_ms": 9.316}, "gstreamer_push": {"avg_ms": 1.117, "count": 17, "max_ms": 1.215, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.354, "count": 17, "max_ms": 51.825, "p95_ms": 51.825}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.951, "count": 17, "max_ms": 53.13, "p95_ms": 53.13}, "yolo11_640_fire_11class": {"avg_ms": 49.335, "count": 17, "max_ms": 51.835, "p95_ms": 51.835}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.477, "count": 17, "max_ms": 50.809, "p95_ms": 50.809}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.575, "count": 17, "max_ms": 100.049, "p95_ms": 100.049}, "read_interval": {"avg_ms": 1872.045, "count": 16, "max_ms": 16335.901, "p95_ms": 16335.901}, "triton_wait": {"avg_ms": 50.163, "count": 17, "max_ms": 53.138, "p95_ms": 53.138}, "visualize": {"avg_ms": 3.578, "count": 17, "max_ms": 4.544, "p95_ms": 4.544}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20069}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16296.227, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:25:26+09:00", "last_error_summary": "[h264 @ 0x55c42a5ed000] error while decoding MB 56 39, bytestream -5 [in#0/rtsp @ 0x55c42a5a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c42a5a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:25:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.286, "last_started_at": "2026-07-06T06:21:33+09:00", "pid": 149241, "process": {"exists": false, "pid": 149241}, "restart_count": 144, "restart_count_per_hour": 3, "rtsp_timeout_count": 246, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.799, "sample_count": 2228, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2408384, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:25:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T06:25:26+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-06T06:25:26+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-06T06:25:26.784351+09:00", "mtime_epoch": 1783286726.7843513, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 179372}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:25:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2590.371, "name": "celery", "pid": 267, "rss_mb": 2590.371, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 17, "pushed_frames": 17, "read_frames": 16, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.567, "pushed": 0.567, "read": 0.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.302, "count": 17, "max_ms": 9.316, "p95_ms": 9.316}, "gstreamer_push": {"avg_ms": 1.117, "count": 17, "max_ms": 1.215, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.354, "count": 17, "max_ms": 51.825, "p95_ms": 51.825}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.951, "count": 17, "max_ms": 53.13, "p95_ms": 53.13}, "yolo11_640_fire_11class": {"avg_ms": 49.335, "count": 17, "max_ms": 51.835, "p95_ms": 51.835}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.477, "count": 17, "max_ms": 50.809, "p95_ms": 50.809}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.575, "count": 17, "max_ms": 100.049, "p95_ms": 100.049}, "read_interval": {"avg_ms": 1872.045, "count": 16, "max_ms": 16335.901, "p95_ms": 16335.901}, "triton_wait": {"avg_ms": 50.163, "count": 17, "max_ms": 53.138, "p95_ms": 53.138}, "visualize": {"avg_ms": 3.578, "count": 17, "max_ms": 4.544, "p95_ms": 4.544}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 91, "tracking_summary": 20069}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16431.305, "limit_mb": 8796093022207.996}, "detail": {"pid": 149667, "restart_count": 145, "start_count": 146}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:25:26+09:00", "last_error_summary": "[h264 @ 0x55c42a5ed000] error while decoding MB 56 39, bytestream -5 [in#0/rtsp @ 0x55c42a5a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c42a5a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:25:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:25:29+09:00", "pid": 149667, "process": {"exists": true, "fd_count": 5, "hwm_mb": 163.277, "name": "ffmpeg", "pid": 149667, "rss_mb": 163.277, "state": "R (running)", "threads": 106}, "restart_count": 145, "restart_count_per_hour": 4, "rtsp_timeout_count": 246, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.793, "sample_count": 2183, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2408384, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:25:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.725, "last_frame_pushed_at": "2026-07-06T06:25:26+09:00", "last_frame_read_age_sec": 4.797, "last_frame_read_at": "2026-07-06T06:25:26+09:00", "mjpeg": {"age_sec": 4.711, "exists": true, "mtime": "2026-07-06T06:25:26.864352+09:00", "mtime_epoch": 1783286726.8643525, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 179210}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:25:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2590.371, "name": "celery", "pid": 267, "rss_mb": 2590.371, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.59, "pushed": 0.59, "read": 0.59}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.377, "count": 13, "max_ms": 9.316, "p95_ms": 9.316}, "gstreamer_push": {"avg_ms": 1.114, "count": 13, "max_ms": 1.215, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.761, "count": 13, "max_ms": 51.825, "p95_ms": 51.825}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.102, "count": 13, "max_ms": 53.13, "p95_ms": 53.13}, "yolo11_640_fire_11class": {"avg_ms": 49.435, "count": 13, "max_ms": 51.835, "p95_ms": 51.835}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.597, "count": 13, "max_ms": 50.809, "p95_ms": 50.809}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.506, "count": 13, "max_ms": 100.049, "p95_ms": 100.049}, "read_interval": {"avg_ms": 2285.548, "count": 13, "max_ms": 16335.901, "p95_ms": 16335.901}, "triton_wait": {"avg_ms": 50.224, "count": 13, "max_ms": 53.138, "p95_ms": 53.138}, "visualize": {"avg_ms": 3.554, "count": 13, "max_ms": 4.544, "p95_ms": 4.544}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.035, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 372, "tracking_summary": 16092}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16361.52, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5616fea7ee80] error while decoding MB 57 42, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 803, "h264_decode_error_count": 1952, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:26:39+09:00", "last_error_summary": "[h264 @ 0x5616fea7ee80] error while decoding MB 57 42, bytestream -15", "last_exit_at": "2026-07-06T06:24:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.366, "last_started_at": "2026-07-06T06:24:31+09:00", "pid": 149560, "process": {"exists": false, "pid": 149560}, "restart_count": 894, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 895}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1113.657, "sample_count": 390, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931144, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:26:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T06:26:39+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T06:26:39+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T06:26:39.177320+09:00", "mtime_epoch": 1783286799.1773198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:26:39+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 43, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.462, "pushed": 1.462, "read": 1.462}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 60.876, "count": 43, "max_ms": 71.21, "p95_ms": 70.017}, "gstreamer_push": {"avg_ms": 0.858, "count": 43, "max_ms": 0.902, "p95_ms": 0.888}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.836, "count": 43, "max_ms": 64.014, "p95_ms": 61.689}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.139, "count": 43, "max_ms": 94.67, "p95_ms": 91.106}, "yolo11_640_fire_11class": {"avg_ms": 50.124, "count": 43, "max_ms": 70.181, "p95_ms": 67.441}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.041, "count": 43, "max_ms": 97.885, "p95_ms": 95.182}}, "truncated_count": 0}, "processing_total": {"avg_ms": 687.811, "count": 43, "max_ms": 804.492, "p95_ms": 767.346}, "read_interval": {"avg_ms": 697.216, "count": 43, "max_ms": 814.006, "p95_ms": 777.362}, "triton_wait": {"avg_ms": 77.912, "count": 43, "max_ms": 97.897, "p95_ms": 95.194}, "visualize": {"avg_ms": 17.453, "count": 43, "max_ms": 23.114, "p95_ms": 21.326}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.417, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 372, "tracking_summary": 16092}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16360.48, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:26:39+09:00", "last_error_summary": "[h264 @ 0x5616fea7ee80] error while decoding MB 57 42, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 803, "h264_decode_error_count": 1952, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:26:39+09:00", "last_error_summary": "[h264 @ 0x5616fea7ee80] error while decoding MB 57 42, bytestream -15", "last_exit_at": "2026-07-06T06:26:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.004, "last_started_at": "2026-07-06T06:24:31+09:00", "pid": 149560, "process": {"exists": false, "pid": 149560}, "restart_count": 894, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 895}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1113.657, "sample_count": 389, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:26:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:26:39+09:00", "last_frame_read_age_sec": 0.687, "last_frame_read_at": "2026-07-06T06:26:39+09:00", "mjpeg": {"age_sec": 0.679, "exists": true, "mtime": "2026-07-06T06:26:39.177320+09:00", "mtime_epoch": 1783286799.1773198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:26:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.465, "pushed": 1.465, "read": 1.43}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 60.936, "count": 43, "max_ms": 71.21, "p95_ms": 70.017}, "gstreamer_push": {"avg_ms": 0.858, "count": 43, "max_ms": 0.902, "p95_ms": 0.888}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.8, "count": 43, "max_ms": 64.014, "p95_ms": 61.689}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.604, "count": 43, "max_ms": 94.67, "p95_ms": 91.106}, "yolo11_640_fire_11class": {"avg_ms": 50.127, "count": 43, "max_ms": 70.181, "p95_ms": 67.441}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.305, "count": 43, "max_ms": 97.885, "p95_ms": 95.182}}, "truncated_count": 0}, "processing_total": {"avg_ms": 689.431, "count": 43, "max_ms": 804.492, "p95_ms": 767.346}, "read_interval": {"avg_ms": 699.018, "count": 42, "max_ms": 814.006, "p95_ms": 777.362}, "triton_wait": {"avg_ms": 78.175, "count": 43, "max_ms": 97.897, "p95_ms": 95.194}, "visualize": {"avg_ms": 17.492, "count": 43, "max_ms": 23.114, "p95_ms": 21.326}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.36, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 372, "tracking_summary": 16092}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16360.703, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 803, "h264_decode_error_count": 1952, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:26:39+09:00", "last_error_summary": "[h264 @ 0x5616fea7ee80] error while decoding MB 57 42, bytestream -15", "last_exit_at": "2026-07-06T06:26:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.007, "last_started_at": "2026-07-06T06:24:31+09:00", "pid": 149560, "process": {"exists": false, "pid": 149560}, "restart_count": 894, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 895}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1113.657, "sample_count": 389, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:26:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:26:39+09:00", "last_frame_read_age_sec": 0.69, "last_frame_read_at": "2026-07-06T06:26:39+09:00", "mjpeg": {"age_sec": 0.682, "exists": true, "mtime": "2026-07-06T06:26:39.177320+09:00", "mtime_epoch": 1783286799.1773198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:26:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.464, "pushed": 1.464, "read": 1.43}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 60.936, "count": 43, "max_ms": 71.21, "p95_ms": 70.017}, "gstreamer_push": {"avg_ms": 0.858, "count": 43, "max_ms": 0.902, "p95_ms": 0.888}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.8, "count": 43, "max_ms": 64.014, "p95_ms": 61.689}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.604, "count": 43, "max_ms": 94.67, "p95_ms": 91.106}, "yolo11_640_fire_11class": {"avg_ms": 50.127, "count": 43, "max_ms": 70.181, "p95_ms": 67.441}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.305, "count": 43, "max_ms": 97.885, "p95_ms": 95.182}}, "truncated_count": 0}, "processing_total": {"avg_ms": 689.431, "count": 43, "max_ms": 804.492, "p95_ms": 767.346}, "read_interval": {"avg_ms": 699.018, "count": 42, "max_ms": 814.006, "p95_ms": 777.362}, "triton_wait": {"avg_ms": 78.175, "count": 43, "max_ms": 97.897, "p95_ms": 95.194}, "visualize": {"avg_ms": 17.492, "count": 43, "max_ms": 23.114, "p95_ms": 21.326}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.363, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 372, "tracking_summary": 16092}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16466.957, "limit_mb": 8796093022207.996}, "detail": {"pid": 149819, "restart_count": 895, "start_count": 896}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 803, "h264_decode_error_count": 1952, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:26:39+09:00", "last_error_summary": "[h264 @ 0x5616fea7ee80] error while decoding MB 57 42, bytestream -15", "last_exit_at": "2026-07-06T06:26:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:26:42+09:00", "pid": 149819, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.746, "name": "ffmpeg", "pid": 149819, "rss_mb": 136.746, "state": "S (sleeping)", "threads": 106}, "restart_count": 895, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 896}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1113.657, "sample_count": 379, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:26:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.252, "last_frame_pushed_at": "2026-07-06T06:26:39+09:00", "last_frame_read_age_sec": 4.936, "last_frame_read_at": "2026-07-06T06:26:39+09:00", "mjpeg": {"age_sec": 4.236, "exists": true, "mtime": "2026-07-06T06:26:39.869329+09:00", "mtime_epoch": 1783286799.869329, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192402}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:26:44+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 37, "pushed_frames": 37, "read_frames": 36, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.248, "pushed": 1.248, "read": 1.214}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 62.029, "count": 36, "max_ms": 71.21, "p95_ms": 70.717}, "gstreamer_push": {"avg_ms": 0.861, "count": 37, "max_ms": 0.902, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.693, "count": 36, "max_ms": 64.014, "p95_ms": 63.492}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.046, "count": 36, "max_ms": 94.67, "p95_ms": 91.106}, "yolo11_640_fire_11class": {"avg_ms": 50.055, "count": 36, "max_ms": 70.181, "p95_ms": 67.657}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.159, "count": 36, "max_ms": 97.258, "p95_ms": 95.182}}, "truncated_count": 0}, "processing_total": {"avg_ms": 695.066, "count": 37, "max_ms": 804.492, "p95_ms": 775.119}, "read_interval": {"avg_ms": 704.881, "count": 36, "max_ms": 814.006, "p95_ms": 783.828}, "triton_wait": {"avg_ms": 77.495, "count": 36, "max_ms": 97.27, "p95_ms": 95.194}, "visualize": {"avg_ms": 17.526, "count": 36, "max_ms": 23.114, "p95_ms": 21.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.655, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 373, "tracking_summary": 16093}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16370.66, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5622bc508840] error while decoding MB 59 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 804, "h264_decode_error_count": 1953, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:29:22+09:00", "last_error_summary": "[h264 @ 0x5622bc508840] error while decoding MB 59 37, bytestream -5", "last_exit_at": "2026-07-06T06:26:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.725, "last_started_at": "2026-07-06T06:26:42+09:00", "pid": 149819, "process": {"exists": false, "pid": 149819}, "restart_count": 895, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 896}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1314.24, "sample_count": 288, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931277, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:29:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.054, "last_frame_pushed_at": "2026-07-06T06:29:22+09:00", "last_frame_read_age_sec": 0.047, "last_frame_read_at": "2026-07-06T06:29:22+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T06:29:22.788426+09:00", "mtime_epoch": 1783286962.788426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193656}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:29:22+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 28, "pushed_frames": 28, "read_frames": 28, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.954, "pushed": 0.954, "read": 0.954}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 102.048, "count": 27, "max_ms": 110.773, "p95_ms": 107.427}, "gstreamer_push": {"avg_ms": 0.692, "count": 28, "max_ms": 0.883, "p95_ms": 0.868}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.218, "count": 27, "max_ms": 60.303, "p95_ms": 59.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.907, "count": 27, "max_ms": 96.902, "p95_ms": 86.989}, "yolo11_640_fire_11class": {"avg_ms": 48.546, "count": 27, "max_ms": 65.723, "p95_ms": 64.545}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.292, "count": 27, "max_ms": 85.809, "p95_ms": 83.221}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1084.951, "count": 28, "max_ms": 1306.42, "p95_ms": 1300.923}, "read_interval": {"avg_ms": 1093.358, "count": 28, "max_ms": 1315.938, "p95_ms": 1310.563}, "triton_wait": {"avg_ms": 75.081, "count": 27, "max_ms": 96.913, "p95_ms": 87.0}, "visualize": {"avg_ms": 15.219, "count": 27, "max_ms": 17.645, "p95_ms": 15.822}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.355, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 373, "tracking_summary": 16093}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16369.621, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:29:22+09:00", "last_error_summary": "[h264 @ 0x5622bc508840] error while decoding MB 59 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 804, "h264_decode_error_count": 1953, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:29:22+09:00", "last_error_summary": "[h264 @ 0x5622bc508840] error while decoding MB 59 37, bytestream -5", "last_exit_at": "2026-07-06T06:29:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.025, "last_started_at": "2026-07-06T06:26:42+09:00", "pid": 149819, "process": {"exists": false, "pid": 149819}, "restart_count": 895, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 896}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1317.169, "sample_count": 287, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931278, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:29:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:29:24+09:00", "last_frame_read_age_sec": 1.347, "last_frame_read_at": "2026-07-06T06:29:22+09:00", "mjpeg": {"age_sec": 1.34, "exists": true, "mtime": "2026-07-06T06:29:22.788426+09:00", "mtime_epoch": 1783286962.788426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193656}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:29:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 28, "pushed_frames": 28, "read_frames": 27, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.954, "pushed": 0.954, "read": 0.92}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 102.669, "count": 27, "max_ms": 116.299, "p95_ms": 110.773}, "gstreamer_push": {"avg_ms": 0.695, "count": 28, "max_ms": 0.908, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.165, "count": 27, "max_ms": 60.303, "p95_ms": 59.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.617, "count": 27, "max_ms": 96.902, "p95_ms": 86.989}, "yolo11_640_fire_11class": {"avg_ms": 47.98, "count": 27, "max_ms": 65.723, "p95_ms": 64.308}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.444, "count": 27, "max_ms": 83.221, "p95_ms": 83.028}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1086.246, "count": 28, "max_ms": 1342.661, "p95_ms": 1300.923}, "read_interval": {"avg_ms": 1085.115, "count": 27, "max_ms": 1310.563, "p95_ms": 1304.081}, "triton_wait": {"avg_ms": 74.629, "count": 27, "max_ms": 96.913, "p95_ms": 87.0}, "visualize": {"avg_ms": 15.475, "count": 27, "max_ms": 22.342, "p95_ms": 17.645}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.353, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 373, "tracking_summary": 16093}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16369.613, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 804, "h264_decode_error_count": 1953, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:29:22+09:00", "last_error_summary": "[h264 @ 0x5622bc508840] error while decoding MB 59 37, bytestream -5", "last_exit_at": "2026-07-06T06:29:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.027, "last_started_at": "2026-07-06T06:26:42+09:00", "pid": 149819, "process": {"exists": false, "pid": 149819}, "restart_count": 895, "restart_count_per_hour": 23, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 896}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1317.169, "sample_count": 287, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931278, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:29:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:29:24+09:00", "last_frame_read_age_sec": 1.35, "last_frame_read_at": "2026-07-06T06:29:22+09:00", "mjpeg": {"age_sec": 1.343, "exists": true, "mtime": "2026-07-06T06:29:22.788426+09:00", "mtime_epoch": 1783286962.788426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193656}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:29:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 28, "pushed_frames": 28, "read_frames": 27, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.954, "pushed": 0.954, "read": 0.92}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 102.669, "count": 27, "max_ms": 116.299, "p95_ms": 110.773}, "gstreamer_push": {"avg_ms": 0.695, "count": 28, "max_ms": 0.908, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.165, "count": 27, "max_ms": 60.303, "p95_ms": 59.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.617, "count": 27, "max_ms": 96.902, "p95_ms": 86.989}, "yolo11_640_fire_11class": {"avg_ms": 47.98, "count": 27, "max_ms": 65.723, "p95_ms": 64.308}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.444, "count": 27, "max_ms": 83.221, "p95_ms": 83.028}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1086.246, "count": 28, "max_ms": 1342.661, "p95_ms": 1300.923}, "read_interval": {"avg_ms": 1085.115, "count": 27, "max_ms": 1310.563, "p95_ms": 1304.081}, "triton_wait": {"avg_ms": 74.629, "count": 27, "max_ms": 96.913, "p95_ms": 87.0}, "visualize": {"avg_ms": 15.475, "count": 27, "max_ms": 22.342, "p95_ms": 17.645}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.355, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 373, "tracking_summary": 16093}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16475.082, "limit_mb": 8796093022207.996}, "detail": {"pid": 149986, "restart_count": 896, "start_count": 897}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 804, "h264_decode_error_count": 1953, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:29:22+09:00", "last_error_summary": "[h264 @ 0x5622bc508840] error while decoding MB 59 37, bytestream -5", "last_exit_at": "2026-07-06T06:29:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:29:26+09:00", "pid": 149986, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.465, "name": "ffmpeg", "pid": 149986, "rss_mb": 136.465, "state": "S (sleeping)", "threads": 106}, "restart_count": 896, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 897}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1317.169, "sample_count": 281, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931278, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:29:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.632, "last_frame_pushed_at": "2026-07-06T06:29:24+09:00", "last_frame_read_age_sec": 5.976, "last_frame_read_at": "2026-07-06T06:29:22+09:00", "mjpeg": {"age_sec": 4.617, "exists": true, "mtime": "2026-07-06T06:29:24.140482+09:00", "mtime_epoch": 1783286964.140482, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193622}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:29:28+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9546.934, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 24, "pushed_frames": 24, "read_frames": 23, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.82, "pushed": 0.82, "read": 0.786}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 102.987, "count": 23, "max_ms": 116.299, "p95_ms": 110.773}, "gstreamer_push": {"avg_ms": 0.677, "count": 24, "max_ms": 0.908, "p95_ms": 0.84}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.111, "count": 23, "max_ms": 60.303, "p95_ms": 59.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.911, "count": 23, "max_ms": 96.902, "p95_ms": 86.989}, "yolo11_640_fire_11class": {"avg_ms": 48.05, "count": 23, "max_ms": 65.723, "p95_ms": 64.308}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.406, "count": 23, "max_ms": 83.221, "p95_ms": 83.028}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1062.958, "count": 24, "max_ms": 1342.661, "p95_ms": 1146.58}, "read_interval": {"avg_ms": 1058.92, "count": 23, "max_ms": 1154.629, "p95_ms": 1125.514}, "triton_wait": {"avg_ms": 74.923, "count": 23, "max_ms": 96.913, "p95_ms": 87.0}, "visualize": {"avg_ms": 15.53, "count": 23, "max_ms": 22.342, "p95_ms": 17.645}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.264, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16097}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16378.395, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557fff3c2500] error while decoding MB 2 15, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 805, "h264_decode_error_count": 1954, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:31:28+09:00", "last_error_summary": "[h264 @ 0x557fff3c2500] error while decoding MB 2 15, bytestream -12", "last_exit_at": "2026-07-06T06:29:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.036, "last_started_at": "2026-07-06T06:29:26+09:00", "pid": 149986, "process": {"exists": false, "pid": 149986}, "restart_count": 896, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 897}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1287.258, "sample_count": 543, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931672, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:31:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T06:31:28+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-06T06:31:28+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T06:31:28.753317+09:00", "mtime_epoch": 1783287088.7533166, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176925}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:31:28+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.191, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.157, "pushed": 3.157, "read": 3.157}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.362, "count": 94, "max_ms": 40.13, "p95_ms": 37.055}, "gstreamer_push": {"avg_ms": 0.858, "count": 94, "max_ms": 0.915, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.64, "count": 94, "max_ms": 73.031, "p95_ms": 62.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.263, "count": 94, "max_ms": 96.017, "p95_ms": 88.56}, "yolo11_640_fire_11class": {"avg_ms": 48.602, "count": 94, "max_ms": 71.054, "p95_ms": 67.028}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.3, "count": 94, "max_ms": 92.831, "p95_ms": 88.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.519, "count": 94, "max_ms": 396.296, "p95_ms": 347.777}, "read_interval": {"avg_ms": 319.635, "count": 94, "max_ms": 404.903, "p95_ms": 357.302}, "triton_wait": {"avg_ms": 68.836, "count": 94, "max_ms": 96.029, "p95_ms": 90.395}, "visualize": {"avg_ms": 10.705, "count": 94, "max_ms": 13.325, "p95_ms": 12.23}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.78, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16097}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16377.211, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:31:28+09:00", "last_error_summary": "[h264 @ 0x557fff3c2500] error while decoding MB 2 15, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 805, "h264_decode_error_count": 1954, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:31:28+09:00", "last_error_summary": "[h264 @ 0x557fff3c2500] error while decoding MB 2 15, bytestream -12", "last_exit_at": "2026-07-06T06:31:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.301, "last_started_at": "2026-07-06T06:29:26+09:00", "pid": 149986, "process": {"exists": false, "pid": 149986}, "restart_count": 896, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 897}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1287.258, "sample_count": 544, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931673, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:31:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:31:29+09:00", "last_frame_read_age_sec": 0.314, "last_frame_read_at": "2026-07-06T06:31:28+09:00", "mjpeg": {"age_sec": 0.304, "exists": true, "mtime": "2026-07-06T06:31:28.753317+09:00", "mtime_epoch": 1783287088.7533166, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176925}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:31:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.191, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.163, "pushed": 3.163, "read": 3.129}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.362, "count": 94, "max_ms": 40.13, "p95_ms": 37.055}, "gstreamer_push": {"avg_ms": 0.858, "count": 94, "max_ms": 0.915, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.809, "count": 94, "max_ms": 73.031, "p95_ms": 62.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.392, "count": 94, "max_ms": 96.017, "p95_ms": 88.56}, "yolo11_640_fire_11class": {"avg_ms": 48.845, "count": 94, "max_ms": 71.054, "p95_ms": 67.216}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.559, "count": 94, "max_ms": 92.831, "p95_ms": 88.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.451, "count": 94, "max_ms": 396.296, "p95_ms": 347.777}, "read_interval": {"avg_ms": 319.577, "count": 93, "max_ms": 404.903, "p95_ms": 357.302}, "triton_wait": {"avg_ms": 69.095, "count": 94, "max_ms": 96.029, "p95_ms": 90.395}, "visualize": {"avg_ms": 10.692, "count": 94, "max_ms": 13.325, "p95_ms": 12.23}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.722, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16097}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16377.203, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 805, "h264_decode_error_count": 1954, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:31:28+09:00", "last_error_summary": "[h264 @ 0x557fff3c2500] error while decoding MB 2 15, bytestream -12", "last_exit_at": "2026-07-06T06:31:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.304, "last_started_at": "2026-07-06T06:29:26+09:00", "pid": 149986, "process": {"exists": false, "pid": 149986}, "restart_count": 896, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 897}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1287.258, "sample_count": 544, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931673, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:31:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:31:29+09:00", "last_frame_read_age_sec": 0.317, "last_frame_read_at": "2026-07-06T06:31:28+09:00", "mjpeg": {"age_sec": 0.307, "exists": true, "mtime": "2026-07-06T06:31:28.753317+09:00", "mtime_epoch": 1783287088.7533166, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176925}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:31:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.191, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.162, "pushed": 3.162, "read": 3.129}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.362, "count": 94, "max_ms": 40.13, "p95_ms": 37.055}, "gstreamer_push": {"avg_ms": 0.858, "count": 94, "max_ms": 0.915, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.809, "count": 94, "max_ms": 73.031, "p95_ms": 62.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.392, "count": 94, "max_ms": 96.017, "p95_ms": 88.56}, "yolo11_640_fire_11class": {"avg_ms": 48.845, "count": 94, "max_ms": 71.054, "p95_ms": 67.216}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.559, "count": 94, "max_ms": 92.831, "p95_ms": 88.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.451, "count": 94, "max_ms": 396.296, "p95_ms": 347.777}, "read_interval": {"avg_ms": 319.577, "count": 93, "max_ms": 404.903, "p95_ms": 357.302}, "triton_wait": {"avg_ms": 69.095, "count": 94, "max_ms": 96.029, "p95_ms": 90.395}, "visualize": {"avg_ms": 10.692, "count": 94, "max_ms": 13.325, "p95_ms": 12.23}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.725, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16097}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16486.32, "limit_mb": 8796093022207.996}, "detail": {"pid": 150153, "restart_count": 897, "start_count": 898}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 805, "h264_decode_error_count": 1954, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:31:28+09:00", "last_error_summary": "[h264 @ 0x557fff3c2500] error while decoding MB 2 15, bytestream -12", "last_exit_at": "2026-07-06T06:31:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:31:31+09:00", "pid": 150153, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.992, "name": "ffmpeg", "pid": 150153, "rss_mb": 138.992, "state": "S (sleeping)", "threads": 106}, "restart_count": 897, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 898}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1287.258, "sample_count": 537, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1931673, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:31:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.724, "last_frame_pushed_at": "2026-07-06T06:31:29+09:00", "last_frame_read_age_sec": 5.035, "last_frame_read_at": "2026-07-06T06:31:28+09:00", "mjpeg": {"age_sec": 4.71, "exists": true, "mtime": "2026-07-06T06:31:29.069328+09:00", "mtime_epoch": 1783287089.0693283, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176878}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:31:33+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.191, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 79, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.677, "pushed": 2.677, "read": 2.644}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.464, "count": 79, "max_ms": 40.13, "p95_ms": 37.055}, "gstreamer_push": {"avg_ms": 0.858, "count": 80, "max_ms": 0.915, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.07, "count": 79, "max_ms": 73.031, "p95_ms": 63.912}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.916, "count": 79, "max_ms": 96.017, "p95_ms": 89.289}, "yolo11_640_fire_11class": {"avg_ms": 49.501, "count": 79, "max_ms": 71.054, "p95_ms": 68.925}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.608, "count": 79, "max_ms": 92.831, "p95_ms": 89.459}}, "truncated_count": 0}, "processing_total": {"avg_ms": 309.629, "count": 80, "max_ms": 396.296, "p95_ms": 347.777}, "read_interval": {"avg_ms": 318.858, "count": 79, "max_ms": 404.903, "p95_ms": 359.905}, "triton_wait": {"avg_ms": 69.334, "count": 79, "max_ms": 96.029, "p95_ms": 90.86}, "visualize": {"avg_ms": 10.608, "count": 79, "max_ms": 13.325, "p95_ms": 12.23}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16380.0, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561f919095c0] error while decoding MB 5 33, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 806, "h264_decode_error_count": 1955, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:33:25+09:00", "last_error_summary": "[h264 @ 0x561f919095c0] error while decoding MB 5 33, bytestream -11", "last_exit_at": "2026-07-06T06:31:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.975, "last_started_at": "2026-07-06T06:31:31+09:00", "pid": 150153, "process": {"exists": false, "pid": 150153}, "restart_count": 897, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 898}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1019.889, "sample_count": 870, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932096, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:33:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T06:33:25+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-06T06:33:25+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T06:33:25.709349+09:00", "mtime_epoch": 1783287205.7093492, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167489}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:33:25+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.782, "pushed": 3.782, "read": 3.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.462, "count": 112, "max_ms": 44.17, "p95_ms": 34.411}, "gstreamer_push": {"avg_ms": 0.871, "count": 113, "max_ms": 1.081, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.433, "count": 112, "max_ms": 66.092, "p95_ms": 61.521}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.501, "count": 112, "max_ms": 92.669, "p95_ms": 87.034}, "yolo11_640_fire_11class": {"avg_ms": 51.542, "count": 112, "max_ms": 71.897, "p95_ms": 67.745}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.503, "count": 112, "max_ms": 91.291, "p95_ms": 87.598}}, "truncated_count": 0}, "processing_total": {"avg_ms": 256.773, "count": 113, "max_ms": 306.111, "p95_ms": 287.803}, "read_interval": {"avg_ms": 266.466, "count": 113, "max_ms": 315.413, "p95_ms": 297.645}, "triton_wait": {"avg_ms": 69.062, "count": 112, "max_ms": 92.681, "p95_ms": 87.937}, "visualize": {"avg_ms": 9.794, "count": 112, "max_ms": 12.871, "p95_ms": 12.216}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.875, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16379.066, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:33:25+09:00", "last_error_summary": "[h264 @ 0x561f919095c0] error while decoding MB 5 33, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 806, "h264_decode_error_count": 1955, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:33:25+09:00", "last_error_summary": "[h264 @ 0x561f919095c0] error while decoding MB 5 33, bytestream -11", "last_exit_at": "2026-07-06T06:33:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.211, "last_started_at": "2026-07-06T06:31:31+09:00", "pid": 150153, "process": {"exists": false, "pid": 150153}, "restart_count": 897, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 898}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1014.035, "sample_count": 870, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932097, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:33:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:33:25+09:00", "last_frame_read_age_sec": 0.284, "last_frame_read_at": "2026-07-06T06:33:25+09:00", "mjpeg": {"age_sec": 0.279, "exists": true, "mtime": "2026-07-06T06:33:25.709349+09:00", "mtime_epoch": 1783287205.7093492, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167489}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:33:25+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.788, "pushed": 3.788, "read": 3.754}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.462, "count": 113, "max_ms": 44.17, "p95_ms": 34.411}, "gstreamer_push": {"avg_ms": 0.87, "count": 113, "max_ms": 1.081, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.546, "count": 113, "max_ms": 66.092, "p95_ms": 61.613}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.605, "count": 113, "max_ms": 92.669, "p95_ms": 87.034}, "yolo11_640_fire_11class": {"avg_ms": 51.677, "count": 113, "max_ms": 71.897, "p95_ms": 67.745}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.664, "count": 113, "max_ms": 91.291, "p95_ms": 87.598}}, "truncated_count": 0}, "processing_total": {"avg_ms": 256.729, "count": 113, "max_ms": 306.111, "p95_ms": 287.803}, "read_interval": {"avg_ms": 266.213, "count": 112, "max_ms": 315.413, "p95_ms": 297.645}, "triton_wait": {"avg_ms": 69.209, "count": 113, "max_ms": 92.681, "p95_ms": 87.937}, "visualize": {"avg_ms": 9.791, "count": 113, "max_ms": 12.871, "p95_ms": 12.216}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.831, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16379.152, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 806, "h264_decode_error_count": 1955, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:33:25+09:00", "last_error_summary": "[h264 @ 0x561f919095c0] error while decoding MB 5 33, bytestream -11", "last_exit_at": "2026-07-06T06:33:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.214, "last_started_at": "2026-07-06T06:31:31+09:00", "pid": 150153, "process": {"exists": false, "pid": 150153}, "restart_count": 897, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 898}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1014.035, "sample_count": 870, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932097, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:33:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:33:25+09:00", "last_frame_read_age_sec": 0.288, "last_frame_read_at": "2026-07-06T06:33:25+09:00", "mjpeg": {"age_sec": 0.282, "exists": true, "mtime": "2026-07-06T06:33:25.709349+09:00", "mtime_epoch": 1783287205.7093492, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167489}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:33:25+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.788, "pushed": 3.788, "read": 3.754}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.462, "count": 113, "max_ms": 44.17, "p95_ms": 34.411}, "gstreamer_push": {"avg_ms": 0.87, "count": 113, "max_ms": 1.081, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.546, "count": 113, "max_ms": 66.092, "p95_ms": 61.613}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.605, "count": 113, "max_ms": 92.669, "p95_ms": 87.034}, "yolo11_640_fire_11class": {"avg_ms": 51.677, "count": 113, "max_ms": 71.897, "p95_ms": 67.745}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.664, "count": 113, "max_ms": 91.291, "p95_ms": 87.598}}, "truncated_count": 0}, "processing_total": {"avg_ms": 256.729, "count": 113, "max_ms": 306.111, "p95_ms": 287.803}, "read_interval": {"avg_ms": 266.213, "count": 112, "max_ms": 315.413, "p95_ms": 297.645}, "triton_wait": {"avg_ms": 69.209, "count": 113, "max_ms": 92.681, "p95_ms": 87.937}, "visualize": {"avg_ms": 9.791, "count": 113, "max_ms": 12.871, "p95_ms": 12.216}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.835, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16487.375, "limit_mb": 8796093022207.996}, "detail": {"pid": 150260, "restart_count": 898, "start_count": 899}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 806, "h264_decode_error_count": 1955, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:33:25+09:00", "last_error_summary": "[h264 @ 0x561f919095c0] error while decoding MB 5 33, bytestream -11", "last_exit_at": "2026-07-06T06:33:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:33:28+09:00", "pid": 150260, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.246, "name": "ffmpeg", "pid": 150260, "rss_mb": 139.246, "state": "S (sleeping)", "threads": 106}, "restart_count": 898, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 899}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 998.23, "sample_count": 866, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932097, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:33:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.665, "last_frame_pushed_at": "2026-07-06T06:33:25+09:00", "last_frame_read_age_sec": 4.946, "last_frame_read_at": "2026-07-06T06:33:25+09:00", "mjpeg": {"age_sec": 4.649, "exists": true, "mtime": "2026-07-06T06:33:26.001359+09:00", "mtime_epoch": 1783287206.0013587, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168606}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:33:30+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.217, "pushed": 3.217, "read": 3.183}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.383, "count": 96, "max_ms": 41.462, "p95_ms": 34.411}, "gstreamer_push": {"avg_ms": 0.872, "count": 96, "max_ms": 1.081, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.317, "count": 96, "max_ms": 66.092, "p95_ms": 61.613}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.147, "count": 96, "max_ms": 92.669, "p95_ms": 87.034}, "yolo11_640_fire_11class": {"avg_ms": 51.223, "count": 96, "max_ms": 71.897, "p95_ms": 67.745}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.342, "count": 96, "max_ms": 91.291, "p95_ms": 87.598}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.178, "count": 96, "max_ms": 306.111, "p95_ms": 287.288}, "read_interval": {"avg_ms": 264.587, "count": 95, "max_ms": 315.413, "p95_ms": 297.054}, "triton_wait": {"avg_ms": 68.889, "count": 96, "max_ms": 92.681, "p95_ms": 87.937}, "visualize": {"avg_ms": 9.702, "count": 96, "max_ms": 12.871, "p95_ms": 11.857}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16382.008, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e91390de80] error while decoding MB 4 41, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 807, "h264_decode_error_count": 1956, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:36:33+09:00", "last_error_summary": "[h264 @ 0x55e91390de80] error while decoding MB 4 41, bytestream -5", "last_exit_at": "2026-07-06T06:33:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.663, "last_started_at": "2026-07-06T06:33:28+09:00", "pid": 150260, "process": {"exists": false, "pid": 150260}, "restart_count": 898, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 899}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1124.393, "sample_count": 631, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:36:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T06:36:33+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T06:36:33+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T06:36:33.271041+09:00", "mtime_epoch": 1783287393.2710414, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190051}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:36:33+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.895, "pushed": 0.895, "read": 0.895}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 116.521, "count": 26, "max_ms": 133.437, "p95_ms": 129.856}, "gstreamer_push": {"avg_ms": 0.64, "count": 26, "max_ms": 0.677, "p95_ms": 0.666}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.673, "count": 26, "max_ms": 60.52, "p95_ms": 59.854}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.59, "count": 26, "max_ms": 95.825, "p95_ms": 89.957}, "yolo11_640_fire_11class": {"avg_ms": 51.714, "count": 26, "max_ms": 66.515, "p95_ms": 66.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.786, "count": 26, "max_ms": 98.41, "p95_ms": 96.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1146.942, "count": 26, "max_ms": 1270.769, "p95_ms": 1245.097}, "read_interval": {"avg_ms": 1155.226, "count": 26, "max_ms": 1278.976, "p95_ms": 1253.231}, "triton_wait": {"avg_ms": 80.49, "count": 26, "max_ms": 98.421, "p95_ms": 96.784}, "visualize": {"avg_ms": 15.663, "count": 26, "max_ms": 17.254, "p95_ms": 17.045}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.043, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16380.586, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:36:33+09:00", "last_error_summary": "[h264 @ 0x55e91390de80] error while decoding MB 4 41, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 807, "h264_decode_error_count": 1956, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:36:33+09:00", "last_error_summary": "[h264 @ 0x55e91390de80] error while decoding MB 4 41, bytestream -5", "last_exit_at": "2026-07-06T06:36:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.849, "last_started_at": "2026-07-06T06:33:28+09:00", "pid": 150260, "process": {"exists": false, "pid": 150260}, "restart_count": 898, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 899}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1139.673, "sample_count": 630, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:36:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:36:34+09:00", "last_frame_read_age_sec": 1.238, "last_frame_read_at": "2026-07-06T06:36:33+09:00", "mjpeg": {"age_sec": 1.227, "exists": true, "mtime": "2026-07-06T06:36:33.271041+09:00", "mtime_epoch": 1783287393.2710414, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190051}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:36:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 25, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.892, "pushed": 0.892, "read": 0.858}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 117.294, "count": 25, "max_ms": 133.437, "p95_ms": 129.856}, "gstreamer_push": {"avg_ms": 0.64, "count": 26, "max_ms": 0.677, "p95_ms": 0.666}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.829, "count": 25, "max_ms": 60.52, "p95_ms": 59.854}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.025, "count": 25, "max_ms": 95.825, "p95_ms": 89.957}, "yolo11_640_fire_11class": {"avg_ms": 51.973, "count": 25, "max_ms": 66.515, "p95_ms": 66.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.948, "count": 25, "max_ms": 98.41, "p95_ms": 96.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1154.174, "count": 26, "max_ms": 1270.769, "p95_ms": 1245.097}, "read_interval": {"avg_ms": 1159.261, "count": 25, "max_ms": 1278.976, "p95_ms": 1253.231}, "triton_wait": {"avg_ms": 80.679, "count": 25, "max_ms": 98.421, "p95_ms": 96.784}, "visualize": {"avg_ms": 15.698, "count": 25, "max_ms": 17.254, "p95_ms": 17.045}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.137, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16380.578, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 807, "h264_decode_error_count": 1956, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:36:33+09:00", "last_error_summary": "[h264 @ 0x55e91390de80] error while decoding MB 4 41, bytestream -5", "last_exit_at": "2026-07-06T06:36:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.851, "last_started_at": "2026-07-06T06:33:28+09:00", "pid": 150260, "process": {"exists": false, "pid": 150260}, "restart_count": 898, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 899}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1139.673, "sample_count": 630, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:36:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T06:36:34+09:00", "last_frame_read_age_sec": 1.241, "last_frame_read_at": "2026-07-06T06:36:33+09:00", "mjpeg": {"age_sec": 1.229, "exists": true, "mtime": "2026-07-06T06:36:33.271041+09:00", "mtime_epoch": 1783287393.2710414, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190051}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:36:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 25, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.892, "pushed": 0.892, "read": 0.858}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 117.294, "count": 25, "max_ms": 133.437, "p95_ms": 129.856}, "gstreamer_push": {"avg_ms": 0.64, "count": 26, "max_ms": 0.677, "p95_ms": 0.666}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.829, "count": 25, "max_ms": 60.52, "p95_ms": 59.854}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.025, "count": 25, "max_ms": 95.825, "p95_ms": 89.957}, "yolo11_640_fire_11class": {"avg_ms": 51.973, "count": 25, "max_ms": 66.515, "p95_ms": 66.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.948, "count": 25, "max_ms": 98.41, "p95_ms": 96.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1154.174, "count": 26, "max_ms": 1270.769, "p95_ms": 1245.097}, "read_interval": {"avg_ms": 1159.261, "count": 25, "max_ms": 1278.976, "p95_ms": 1253.231}, "triton_wait": {"avg_ms": 80.679, "count": 25, "max_ms": 98.421, "p95_ms": 96.784}, "visualize": {"avg_ms": 15.698, "count": 25, "max_ms": 17.254, "p95_ms": 17.045}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.14, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 374, "tracking_summary": 16102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16489.426, "limit_mb": 8796093022207.996}, "detail": {"pid": 150367, "restart_count": 899, "start_count": 900}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 807, "h264_decode_error_count": 1956, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:36:33+09:00", "last_error_summary": "[h264 @ 0x55e91390de80] error while decoding MB 4 41, bytestream -5", "last_exit_at": "2026-07-06T06:36:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:36:36+09:00", "pid": 150367, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.867, "name": "ffmpeg", "pid": 150367, "rss_mb": 138.867, "state": "S (sleeping)", "threads": 106}, "restart_count": 899, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 900}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1139.673, "sample_count": 614, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:36:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.157, "last_frame_pushed_at": "2026-07-06T06:36:34+09:00", "last_frame_read_age_sec": 5.392, "last_frame_read_at": "2026-07-06T06:36:33+09:00", "mjpeg": {"age_sec": 4.137, "exists": true, "mtime": "2026-07-06T06:36:34.515077+09:00", "mtime_epoch": 1783287394.5150766, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191042}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:36:38+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 22, "pushed_frames": 22, "read_frames": 21, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.76, "pushed": 0.76, "read": 0.725}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 118.42, "count": 22, "max_ms": 133.437, "p95_ms": 129.856}, "gstreamer_push": {"avg_ms": 0.641, "count": 22, "max_ms": 0.677, "p95_ms": 0.663}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.135, "count": 22, "max_ms": 60.52, "p95_ms": 59.854}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.81, "count": 22, "max_ms": 95.825, "p95_ms": 89.957}, "yolo11_640_fire_11class": {"avg_ms": 52.126, "count": 22, "max_ms": 66.515, "p95_ms": 66.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.952, "count": 22, "max_ms": 98.41, "p95_ms": 94.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1167.931, "count": 22, "max_ms": 1270.769, "p95_ms": 1245.097}, "read_interval": {"avg_ms": 1173.04, "count": 21, "max_ms": 1278.976, "p95_ms": 1253.231}, "triton_wait": {"avg_ms": 80.782, "count": 22, "max_ms": 98.421, "p95_ms": 95.837}, "visualize": {"avg_ms": 15.577, "count": 22, "max_ms": 17.254, "p95_ms": 16.769}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.958, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 386, "tracking_summary": 16103}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16404.117, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561116287600] error while decoding MB 3 24, bytestream -24", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 808, "h264_decode_error_count": 1957, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:38:39+09:00", "last_error_summary": "[h264 @ 0x561116287600] error while decoding MB 3 24, bytestream -24", "last_exit_at": "2026-07-06T06:36:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.218, "last_started_at": "2026-07-06T06:36:36+09:00", "pid": 150367, "process": {"exists": false, "pid": 150367}, "restart_count": 899, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 900}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1196.65, "sample_count": 344, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932460, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:38:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-06T06:38:39+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-06T06:38:39+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-06T06:38:39.830475+09:00", "mtime_epoch": 1783287519.830475, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183144}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:38:39+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 42, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.404, "pushed": 1.404, "read": 1.404}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 62.667, "count": 41, "max_ms": 71.946, "p95_ms": 69.781}, "gstreamer_push": {"avg_ms": 0.853, "count": 42, "max_ms": 0.959, "p95_ms": 0.878}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.587, "count": 41, "max_ms": 68.469, "p95_ms": 64.056}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.39, "count": 41, "max_ms": 92.941, "p95_ms": 90.606}, "yolo11_640_fire_11class": {"avg_ms": 48.464, "count": 41, "max_ms": 62.193, "p95_ms": 61.568}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.066, "count": 41, "max_ms": 89.559, "p95_ms": 85.038}}, "truncated_count": 0}, "processing_total": {"avg_ms": 719.632, "count": 42, "max_ms": 977.483, "p95_ms": 819.024}, "read_interval": {"avg_ms": 728.741, "count": 42, "max_ms": 985.927, "p95_ms": 828.66}, "triton_wait": {"avg_ms": 74.952, "count": 41, "max_ms": 92.954, "p95_ms": 90.619}, "visualize": {"avg_ms": 14.467, "count": 41, "max_ms": 19.232, "p95_ms": 19.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 386, "tracking_summary": 16103}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16403.34, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:38:39+09:00", "last_error_summary": "[h264 @ 0x561116287600] error while decoding MB 3 24, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 808, "h264_decode_error_count": 1957, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:38:39+09:00", "last_error_summary": "[h264 @ 0x561116287600] error while decoding MB 3 24, bytestream -24", "last_exit_at": "2026-07-06T06:38:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.773, "last_started_at": "2026-07-06T06:36:36+09:00", "pid": 150367, "process": {"exists": false, "pid": 150367}, "restart_count": 899, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 900}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1196.65, "sample_count": 344, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932461, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:38:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:38:40+09:00", "last_frame_read_age_sec": 0.603, "last_frame_read_at": "2026-07-06T06:38:39+09:00", "mjpeg": {"age_sec": 0.593, "exists": true, "mtime": "2026-07-06T06:38:39.830475+09:00", "mtime_epoch": 1783287519.830475, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183144}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:38:40+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 41, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.411, "pushed": 1.411, "read": 1.378}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 62.489, "count": 41, "max_ms": 71.946, "p95_ms": 69.781}, "gstreamer_push": {"avg_ms": 0.853, "count": 42, "max_ms": 0.959, "p95_ms": 0.878}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.619, "count": 41, "max_ms": 68.469, "p95_ms": 64.056}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.568, "count": 41, "max_ms": 92.941, "p95_ms": 90.606}, "yolo11_640_fire_11class": {"avg_ms": 48.449, "count": 41, "max_ms": 62.193, "p95_ms": 61.568}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.13, "count": 41, "max_ms": 89.559, "p95_ms": 85.038}}, "truncated_count": 0}, "processing_total": {"avg_ms": 716.453, "count": 42, "max_ms": 977.483, "p95_ms": 819.024}, "read_interval": {"avg_ms": 728.43, "count": 41, "max_ms": 985.927, "p95_ms": 828.66}, "triton_wait": {"avg_ms": 75.13, "count": 41, "max_ms": 92.954, "p95_ms": 90.619}, "visualize": {"avg_ms": 14.534, "count": 41, "max_ms": 19.232, "p95_ms": 19.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.762, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 386, "tracking_summary": 16103}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16403.332, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 808, "h264_decode_error_count": 1957, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:38:39+09:00", "last_error_summary": "[h264 @ 0x561116287600] error while decoding MB 3 24, bytestream -24", "last_exit_at": "2026-07-06T06:38:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.775, "last_started_at": "2026-07-06T06:36:36+09:00", "pid": 150367, "process": {"exists": false, "pid": 150367}, "restart_count": 899, "restart_count_per_hour": 24, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 900}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1196.65, "sample_count": 344, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932461, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:38:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:38:40+09:00", "last_frame_read_age_sec": 0.606, "last_frame_read_at": "2026-07-06T06:38:39+09:00", "mjpeg": {"age_sec": 0.596, "exists": true, "mtime": "2026-07-06T06:38:39.830475+09:00", "mtime_epoch": 1783287519.830475, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183144}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:38:40+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 41, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.411, "pushed": 1.411, "read": 1.377}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 62.489, "count": 41, "max_ms": 71.946, "p95_ms": 69.781}, "gstreamer_push": {"avg_ms": 0.853, "count": 42, "max_ms": 0.959, "p95_ms": 0.878}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.619, "count": 41, "max_ms": 68.469, "p95_ms": 64.056}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.568, "count": 41, "max_ms": 92.941, "p95_ms": 90.606}, "yolo11_640_fire_11class": {"avg_ms": 48.449, "count": 41, "max_ms": 62.193, "p95_ms": 61.568}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.13, "count": 41, "max_ms": 89.559, "p95_ms": 85.038}}, "truncated_count": 0}, "processing_total": {"avg_ms": 716.453, "count": 42, "max_ms": 977.483, "p95_ms": 819.024}, "read_interval": {"avg_ms": 728.43, "count": 41, "max_ms": 985.927, "p95_ms": 828.66}, "triton_wait": {"avg_ms": 75.13, "count": 41, "max_ms": 92.954, "p95_ms": 90.619}, "visualize": {"avg_ms": 14.534, "count": 41, "max_ms": 19.232, "p95_ms": 19.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.765, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 386, "tracking_summary": 16103}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16511.883, "limit_mb": 8796093022207.996}, "detail": {"pid": 150534, "restart_count": 900, "start_count": 901}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 808, "h264_decode_error_count": 1957, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:38:39+09:00", "last_error_summary": "[h264 @ 0x561116287600] error while decoding MB 3 24, bytestream -24", "last_exit_at": "2026-07-06T06:38:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:38:42+09:00", "pid": 150534, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.141, "name": "ffmpeg", "pid": 150534, "rss_mb": 139.141, "state": "S (sleeping)", "threads": 106}, "restart_count": 900, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 901}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1196.65, "sample_count": 336, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932461, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:38:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.327, "last_frame_pushed_at": "2026-07-06T06:38:40+09:00", "last_frame_read_age_sec": 4.926, "last_frame_read_at": "2026-07-06T06:38:39+09:00", "mjpeg": {"age_sec": 4.309, "exists": true, "mtime": "2026-07-06T06:38:40.438491+09:00", "mtime_epoch": 1783287520.4384909, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:38:44+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.449, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 35, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.226, "pushed": 1.226, "read": 1.192}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 62.404, "count": 35, "max_ms": 71.946, "p95_ms": 71.044}, "gstreamer_push": {"avg_ms": 0.85, "count": 36, "max_ms": 0.89, "p95_ms": 0.878}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.483, "count": 35, "max_ms": 68.469, "p95_ms": 68.011}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.804, "count": 35, "max_ms": 92.941, "p95_ms": 90.606}, "yolo11_640_fire_11class": {"avg_ms": 48.42, "count": 35, "max_ms": 62.193, "p95_ms": 62.08}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.319, "count": 35, "max_ms": 89.559, "p95_ms": 82.801}}, "truncated_count": 0}, "processing_total": {"avg_ms": 713.928, "count": 36, "max_ms": 977.483, "p95_ms": 828.679}, "read_interval": {"avg_ms": 726.273, "count": 35, "max_ms": 985.927, "p95_ms": 837.242}, "triton_wait": {"avg_ms": 74.203, "count": 35, "max_ms": 92.954, "p95_ms": 90.619}, "visualize": {"avg_ms": 14.395, "count": 35, "max_ms": 19.05, "p95_ms": 19.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.369, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 391, "tracking_summary": 16105}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16416.168, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5582b83b09c0] error while decoding MB 77 22, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 809, "h264_decode_error_count": 1958, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:40:56+09:00", "last_error_summary": "[h264 @ 0x5582b83b09c0] error while decoding MB 77 22, bytestream -6", "last_exit_at": "2026-07-06T06:38:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.867, "last_started_at": "2026-07-06T06:38:42+09:00", "pid": 150534, "process": {"exists": false, "pid": 150534}, "restart_count": 900, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 901}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1105.612, "sample_count": 424, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932695, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:40:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.054, "last_frame_pushed_at": "2026-07-06T06:40:56+09:00", "last_frame_read_age_sec": 0.047, "last_frame_read_at": "2026-07-06T06:40:56+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T06:40:56.573911+09:00", "mtime_epoch": 1783287656.5739107, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191479}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:40:56+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.707, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.151, "pushed": 2.151, "read": 2.151}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 43.66, "count": 63, "max_ms": 51.099, "p95_ms": 46.952}, "gstreamer_push": {"avg_ms": 0.863, "count": 64, "max_ms": 1.12, "p95_ms": 0.917}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.722, "count": 63, "max_ms": 73.308, "p95_ms": 67.125}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.855, "count": 63, "max_ms": 102.517, "p95_ms": 97.013}, "yolo11_640_fire_11class": {"avg_ms": 52.071, "count": 63, "max_ms": 67.551, "p95_ms": 64.8}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.807, "count": 63, "max_ms": 91.702, "p95_ms": 90.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 462.12, "count": 64, "max_ms": 522.519, "p95_ms": 499.163}, "read_interval": {"avg_ms": 471.56, "count": 64, "max_ms": 532.78, "p95_ms": 508.393}, "triton_wait": {"avg_ms": 80.392, "count": 63, "max_ms": 102.529, "p95_ms": 97.025}, "visualize": {"avg_ms": 13.34, "count": 63, "max_ms": 16.8, "p95_ms": 15.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.755, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 391, "tracking_summary": 16105}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16415.477, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:40:56+09:00", "last_error_summary": "[h264 @ 0x5582b83b09c0] error while decoding MB 77 22, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 809, "h264_decode_error_count": 1958, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:40:56+09:00", "last_error_summary": "[h264 @ 0x5582b83b09c0] error while decoding MB 77 22, bytestream -6", "last_exit_at": "2026-07-06T06:40:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.3, "last_started_at": "2026-07-06T06:38:42+09:00", "pid": 150534, "process": {"exists": false, "pid": 150534}, "restart_count": 900, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 901}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1105.612, "sample_count": 425, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:40:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T06:40:57+09:00", "last_frame_read_age_sec": 0.48, "last_frame_read_at": "2026-07-06T06:40:56+09:00", "mjpeg": {"age_sec": 0.472, "exists": true, "mtime": "2026-07-06T06:40:56.573911+09:00", "mtime_epoch": 1783287656.5739107, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191479}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:40:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.707, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.15, "pushed": 2.15, "read": 2.117}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 43.677, "count": 63, "max_ms": 51.099, "p95_ms": 46.952}, "gstreamer_push": {"avg_ms": 0.863, "count": 64, "max_ms": 1.12, "p95_ms": 0.917}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.767, "count": 63, "max_ms": 73.308, "p95_ms": 67.125}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.839, "count": 63, "max_ms": 102.517, "p95_ms": 97.013}, "yolo11_640_fire_11class": {"avg_ms": 52.066, "count": 63, "max_ms": 67.551, "p95_ms": 64.8}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.734, "count": 63, "max_ms": 91.702, "p95_ms": 90.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 462.159, "count": 64, "max_ms": 522.519, "p95_ms": 499.163}, "read_interval": {"avg_ms": 471.415, "count": 63, "max_ms": 532.78, "p95_ms": 508.393}, "triton_wait": {"avg_ms": 80.377, "count": 63, "max_ms": 102.529, "p95_ms": 97.025}, "visualize": {"avg_ms": 13.369, "count": 63, "max_ms": 16.8, "p95_ms": 15.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.765, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 391, "tracking_summary": 16105}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16415.719, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 809, "h264_decode_error_count": 1958, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:40:56+09:00", "last_error_summary": "[h264 @ 0x5582b83b09c0] error while decoding MB 77 22, bytestream -6", "last_exit_at": "2026-07-06T06:40:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.303, "last_started_at": "2026-07-06T06:38:42+09:00", "pid": 150534, "process": {"exists": false, "pid": 150534}, "restart_count": 900, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 901}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1105.612, "sample_count": 425, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:40:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:40:57+09:00", "last_frame_read_age_sec": 0.482, "last_frame_read_at": "2026-07-06T06:40:56+09:00", "mjpeg": {"age_sec": 0.475, "exists": true, "mtime": "2026-07-06T06:40:56.573911+09:00", "mtime_epoch": 1783287656.5739107, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191479}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:40:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.707, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.15, "pushed": 2.15, "read": 2.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 43.677, "count": 63, "max_ms": 51.099, "p95_ms": 46.952}, "gstreamer_push": {"avg_ms": 0.863, "count": 64, "max_ms": 1.12, "p95_ms": 0.917}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.767, "count": 63, "max_ms": 73.308, "p95_ms": 67.125}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.839, "count": 63, "max_ms": 102.517, "p95_ms": 97.013}, "yolo11_640_fire_11class": {"avg_ms": 52.066, "count": 63, "max_ms": 67.551, "p95_ms": 64.8}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.734, "count": 63, "max_ms": 91.702, "p95_ms": 90.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 462.159, "count": 64, "max_ms": 522.519, "p95_ms": 499.163}, "read_interval": {"avg_ms": 471.415, "count": 63, "max_ms": 532.78, "p95_ms": 508.393}, "triton_wait": {"avg_ms": 80.377, "count": 63, "max_ms": 102.529, "p95_ms": 97.025}, "visualize": {"avg_ms": 13.369, "count": 63, "max_ms": 16.8, "p95_ms": 15.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.768, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 391, "tracking_summary": 16105}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16523.438, "limit_mb": 8796093022207.996}, "detail": {"pid": 150701, "restart_count": 901, "start_count": 902}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 809, "h264_decode_error_count": 1958, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:40:56+09:00", "last_error_summary": "[h264 @ 0x5582b83b09c0] error while decoding MB 77 22, bytestream -6", "last_exit_at": "2026-07-06T06:40:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:40:59+09:00", "pid": 150701, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.074, "name": "ffmpeg", "pid": 150701, "rss_mb": 139.074, "state": "S (sleeping)", "threads": 106}, "restart_count": 901, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 902}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1105.612, "sample_count": 420, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1932696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:40:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.842, "last_frame_pushed_at": "2026-07-06T06:40:57+09:00", "last_frame_read_age_sec": 5.318, "last_frame_read_at": "2026-07-06T06:40:56+09:00", "mjpeg": {"age_sec": 4.826, "exists": true, "mtime": "2026-07-06T06:40:57.057923+09:00", "mtime_epoch": 1783287657.0579226, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191781}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:41:01+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.707, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 53, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.805, "pushed": 1.805, "read": 1.772}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 44.003, "count": 53, "max_ms": 51.099, "p95_ms": 47.104}, "gstreamer_push": {"avg_ms": 0.86, "count": 54, "max_ms": 0.949, "p95_ms": 0.917}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.096, "count": 53, "max_ms": 73.308, "p95_ms": 67.125}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.769, "count": 53, "max_ms": 102.517, "p95_ms": 97.786}, "yolo11_640_fire_11class": {"avg_ms": 52.672, "count": 53, "max_ms": 67.551, "p95_ms": 66.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.73, "count": 53, "max_ms": 91.702, "p95_ms": 90.743}}, "truncated_count": 0}, "processing_total": {"avg_ms": 463.537, "count": 54, "max_ms": 522.519, "p95_ms": 499.163}, "read_interval": {"avg_ms": 472.758, "count": 53, "max_ms": 532.78, "p95_ms": 508.393}, "triton_wait": {"avg_ms": 80.987, "count": 53, "max_ms": 102.529, "p95_ms": 97.799}, "visualize": {"avg_ms": 13.279, "count": 53, "max_ms": 16.8, "p95_ms": 14.899}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 391, "tracking_summary": 16108}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16416.875, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ca36a66940] error while decoding MB 6 40, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 810, "h264_decode_error_count": 1959, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:43:03+09:00", "last_error_summary": "[h264 @ 0x55ca36a66940] error while decoding MB 6 40, bytestream -11", "last_exit_at": "2026-07-06T06:40:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.808, "last_started_at": "2026-07-06T06:40:59+09:00", "pid": 150701, "process": {"exists": false, "pid": 150701}, "restart_count": 901, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 902}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 719.157, "sample_count": 646, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:43:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T06:43:03+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-06T06:43:03+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T06:43:03.648906+09:00", "mtime_epoch": 1783287783.6489055, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183711}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:43:03+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.965, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.512, "pushed": 3.512, "read": 3.512}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.274, "count": 104, "max_ms": 39.122, "p95_ms": 36.158}, "gstreamer_push": {"avg_ms": 0.859, "count": 105, "max_ms": 0.9, "p95_ms": 0.89}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.317, "count": 104, "max_ms": 64.188, "p95_ms": 62.804}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.318, "count": 104, "max_ms": 102.877, "p95_ms": 97.113}, "yolo11_640_fire_11class": {"avg_ms": 51.081, "count": 104, "max_ms": 71.285, "p95_ms": 66.513}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.278, "count": 104, "max_ms": 93.13, "p95_ms": 87.916}}, "truncated_count": 0}, "processing_total": {"avg_ms": 278.058, "count": 105, "max_ms": 335.031, "p95_ms": 323.523}, "read_interval": {"avg_ms": 287.196, "count": 105, "max_ms": 344.804, "p95_ms": 332.746}, "triton_wait": {"avg_ms": 76.945, "count": 104, "max_ms": 102.888, "p95_ms": 97.125}, "visualize": {"avg_ms": 9.763, "count": 104, "max_ms": 12.613, "p95_ms": 11.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 391, "tracking_summary": 16108}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16416.027, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:43:03+09:00", "last_error_summary": "[h264 @ 0x55ca36a66940] error while decoding MB 6 40, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 810, "h264_decode_error_count": 1959, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:43:03+09:00", "last_error_summary": "[h264 @ 0x55ca36a66940] error while decoding MB 6 40, bytestream -11", "last_exit_at": "2026-07-06T06:43:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.068, "last_started_at": "2026-07-06T06:40:59+09:00", "pid": 150701, "process": {"exists": false, "pid": 150701}, "restart_count": 901, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 902}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 707.398, "sample_count": 646, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933057, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:43:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:43:03+09:00", "last_frame_read_age_sec": 0.308, "last_frame_read_at": "2026-07-06T06:43:03+09:00", "mjpeg": {"age_sec": 0.303, "exists": true, "mtime": "2026-07-06T06:43:03.648906+09:00", "mtime_epoch": 1783287783.6489055, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183711}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:43:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.965, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.518, "pushed": 3.518, "read": 3.484}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.269, "count": 104, "max_ms": 39.122, "p95_ms": 36.158}, "gstreamer_push": {"avg_ms": 0.859, "count": 105, "max_ms": 0.9, "p95_ms": 0.89}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.496, "count": 104, "max_ms": 64.188, "p95_ms": 62.804}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.608, "count": 104, "max_ms": 102.877, "p95_ms": 97.939}, "yolo11_640_fire_11class": {"avg_ms": 51.264, "count": 104, "max_ms": 71.285, "p95_ms": 66.513}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.519, "count": 104, "max_ms": 93.13, "p95_ms": 87.916}}, "truncated_count": 0}, "processing_total": {"avg_ms": 278.044, "count": 105, "max_ms": 335.031, "p95_ms": 323.523}, "read_interval": {"avg_ms": 286.928, "count": 104, "max_ms": 344.804, "p95_ms": 332.746}, "triton_wait": {"avg_ms": 77.235, "count": 104, "max_ms": 102.888, "p95_ms": 97.952}, "visualize": {"avg_ms": 9.757, "count": 104, "max_ms": 12.613, "p95_ms": 11.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.851, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 391, "tracking_summary": 16108}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16415.988, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 810, "h264_decode_error_count": 1959, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:43:03+09:00", "last_error_summary": "[h264 @ 0x55ca36a66940] error while decoding MB 6 40, bytestream -11", "last_exit_at": "2026-07-06T06:43:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.072, "last_started_at": "2026-07-06T06:40:59+09:00", "pid": 150701, "process": {"exists": false, "pid": 150701}, "restart_count": 901, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 902}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 707.398, "sample_count": 646, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933057, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:43:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T06:43:03+09:00", "last_frame_read_age_sec": 0.312, "last_frame_read_at": "2026-07-06T06:43:03+09:00", "mjpeg": {"age_sec": 0.306, "exists": true, "mtime": "2026-07-06T06:43:03.648906+09:00", "mtime_epoch": 1783287783.6489055, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183711}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:43:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.965, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.517, "pushed": 3.517, "read": 3.484}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.269, "count": 104, "max_ms": 39.122, "p95_ms": 36.158}, "gstreamer_push": {"avg_ms": 0.859, "count": 105, "max_ms": 0.9, "p95_ms": 0.89}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.496, "count": 104, "max_ms": 64.188, "p95_ms": 62.804}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.608, "count": 104, "max_ms": 102.877, "p95_ms": 97.939}, "yolo11_640_fire_11class": {"avg_ms": 51.264, "count": 104, "max_ms": 71.285, "p95_ms": 66.513}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.519, "count": 104, "max_ms": 93.13, "p95_ms": 87.916}}, "truncated_count": 0}, "processing_total": {"avg_ms": 278.044, "count": 105, "max_ms": 335.031, "p95_ms": 323.523}, "read_interval": {"avg_ms": 286.928, "count": 104, "max_ms": 344.804, "p95_ms": 332.746}, "triton_wait": {"avg_ms": 77.235, "count": 104, "max_ms": 102.888, "p95_ms": 97.952}, "visualize": {"avg_ms": 9.757, "count": 104, "max_ms": 12.613, "p95_ms": 11.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.855, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 391, "tracking_summary": 16108}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16523.441, "limit_mb": 8796093022207.996}, "detail": {"pid": 150808, "restart_count": 902, "start_count": 903}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 810, "h264_decode_error_count": 1959, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:43:03+09:00", "last_error_summary": "[h264 @ 0x55ca36a66940] error while decoding MB 6 40, bytestream -11", "last_exit_at": "2026-07-06T06:43:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:43:06+09:00", "pid": 150808, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.945, "name": "ffmpeg", "pid": 150808, "rss_mb": 138.945, "state": "S (sleeping)", "threads": 106}, "restart_count": 902, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 903}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 704.832, "sample_count": 641, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933057, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:43:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.46, "last_frame_pushed_at": "2026-07-06T06:43:03+09:00", "last_frame_read_age_sec": 4.766, "last_frame_read_at": "2026-07-06T06:43:03+09:00", "mjpeg": {"age_sec": 4.444, "exists": true, "mtime": "2026-07-06T06:43:03.964913+09:00", "mtime_epoch": 1783287783.9649127, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:43:08+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.965, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.029, "pushed": 3.029, "read": 2.995}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.099, "count": 90, "max_ms": 39.122, "p95_ms": 35.965}, "gstreamer_push": {"avg_ms": 0.86, "count": 90, "max_ms": 0.9, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.467, "count": 90, "max_ms": 63.39, "p95_ms": 62.804}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.015, "count": 90, "max_ms": 102.877, "p95_ms": 98.214}, "yolo11_640_fire_11class": {"avg_ms": 51.301, "count": 90, "max_ms": 71.285, "p95_ms": 66.513}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.842, "count": 90, "max_ms": 93.13, "p95_ms": 88.106}}, "truncated_count": 0}, "processing_total": {"avg_ms": 275.13, "count": 90, "max_ms": 328.862, "p95_ms": 319.175}, "read_interval": {"avg_ms": 283.948, "count": 89, "max_ms": 339.167, "p95_ms": 328.182}, "triton_wait": {"avg_ms": 77.672, "count": 90, "max_ms": 102.888, "p95_ms": 98.225}, "visualize": {"avg_ms": 9.658, "count": 90, "max_ms": 12.613, "p95_ms": 11.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.714, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16111}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16434.301, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f359880640] error while decoding MB 4 28, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 811, "h264_decode_error_count": 1960, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:44:57+09:00", "last_error_summary": "[h264 @ 0x55f359880640] error while decoding MB 4 28, bytestream -9", "last_exit_at": "2026-07-06T06:43:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.343, "last_started_at": "2026-07-06T06:43:06+09:00", "pid": 150808, "process": {"exists": false, "pid": 150808}, "restart_count": 902, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 903}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 530.874, "sample_count": 765, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933344, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:44:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T06:44:57+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T06:44:57+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T06:44:57.711467+09:00", "mtime_epoch": 1783287897.7114666, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176581}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:44:57+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.965, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.34, "pushed": 3.34, "read": 3.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.804, "count": 99, "max_ms": 43.274, "p95_ms": 38.823}, "gstreamer_push": {"avg_ms": 0.869, "count": 100, "max_ms": 1.097, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.74, "count": 99, "max_ms": 66.748, "p95_ms": 62.782}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.176, "count": 99, "max_ms": 92.937, "p95_ms": 89.997}, "yolo11_640_fire_11class": {"avg_ms": 49.791, "count": 99, "max_ms": 68.279, "p95_ms": 65.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.546, "count": 99, "max_ms": 95.771, "p95_ms": 92.172}}, "truncated_count": 0}, "processing_total": {"avg_ms": 293.031, "count": 100, "max_ms": 365.519, "p95_ms": 320.891}, "read_interval": {"avg_ms": 302.445, "count": 100, "max_ms": 374.437, "p95_ms": 330.588}, "triton_wait": {"avg_ms": 72.867, "count": 99, "max_ms": 95.783, "p95_ms": 92.185}, "visualize": {"avg_ms": 10.552, "count": 99, "max_ms": 15.769, "p95_ms": 15.051}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16111}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16433.836, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:44:57+09:00", "last_error_summary": "[h264 @ 0x55f359880640] error while decoding MB 4 28, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 811, "h264_decode_error_count": 1960, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:44:57+09:00", "last_error_summary": "[h264 @ 0x55f359880640] error while decoding MB 4 28, bytestream -9", "last_exit_at": "2026-07-06T06:44:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.607, "last_started_at": "2026-07-06T06:43:06+09:00", "pid": 150808, "process": {"exists": false, "pid": 150808}, "restart_count": 902, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 903}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 530.874, "sample_count": 766, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933345, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:44:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T06:44:58+09:00", "last_frame_read_age_sec": 0.316, "last_frame_read_at": "2026-07-06T06:44:57+09:00", "mjpeg": {"age_sec": 0.304, "exists": true, "mtime": "2026-07-06T06:44:57.711467+09:00", "mtime_epoch": 1783287897.7114666, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176581}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:44:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.965, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.348, "pushed": 3.348, "read": 3.315}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.797, "count": 99, "max_ms": 43.274, "p95_ms": 38.823}, "gstreamer_push": {"avg_ms": 0.869, "count": 100, "max_ms": 1.097, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.773, "count": 99, "max_ms": 66.748, "p95_ms": 62.782}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.219, "count": 99, "max_ms": 92.937, "p95_ms": 89.997}, "yolo11_640_fire_11class": {"avg_ms": 49.808, "count": 99, "max_ms": 68.279, "p95_ms": 65.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.675, "count": 99, "max_ms": 95.771, "p95_ms": 92.172}}, "truncated_count": 0}, "processing_total": {"avg_ms": 292.55, "count": 100, "max_ms": 365.519, "p95_ms": 320.427}, "read_interval": {"avg_ms": 301.788, "count": 99, "max_ms": 374.437, "p95_ms": 330.588}, "triton_wait": {"avg_ms": 72.945, "count": 99, "max_ms": 95.783, "p95_ms": 92.185}, "visualize": {"avg_ms": 10.524, "count": 99, "max_ms": 15.769, "p95_ms": 15.051}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16111}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16433.828, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 811, "h264_decode_error_count": 1960, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:44:57+09:00", "last_error_summary": "[h264 @ 0x55f359880640] error while decoding MB 4 28, bytestream -9", "last_exit_at": "2026-07-06T06:44:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.61, "last_started_at": "2026-07-06T06:43:06+09:00", "pid": 150808, "process": {"exists": false, "pid": 150808}, "restart_count": 902, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 903}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 530.874, "sample_count": 766, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933345, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:44:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T06:44:58+09:00", "last_frame_read_age_sec": 0.319, "last_frame_read_at": "2026-07-06T06:44:57+09:00", "mjpeg": {"age_sec": 0.307, "exists": true, "mtime": "2026-07-06T06:44:57.711467+09:00", "mtime_epoch": 1783287897.7114666, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176581}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:44:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.965, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.348, "pushed": 3.348, "read": 3.314}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.797, "count": 99, "max_ms": 43.274, "p95_ms": 38.823}, "gstreamer_push": {"avg_ms": 0.869, "count": 100, "max_ms": 1.097, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.773, "count": 99, "max_ms": 66.748, "p95_ms": 62.782}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.219, "count": 99, "max_ms": 92.937, "p95_ms": 89.997}, "yolo11_640_fire_11class": {"avg_ms": 49.808, "count": 99, "max_ms": 68.279, "p95_ms": 65.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.675, "count": 99, "max_ms": 95.771, "p95_ms": 92.172}}, "truncated_count": 0}, "processing_total": {"avg_ms": 292.55, "count": 100, "max_ms": 365.519, "p95_ms": 320.427}, "read_interval": {"avg_ms": 301.788, "count": 99, "max_ms": 374.437, "p95_ms": 330.588}, "triton_wait": {"avg_ms": 72.945, "count": 99, "max_ms": 95.783, "p95_ms": 92.185}, "visualize": {"avg_ms": 10.524, "count": 99, "max_ms": 15.769, "p95_ms": 15.051}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16111}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16541.738, "limit_mb": 8796093022207.996}, "detail": {"pid": 151035, "restart_count": 903, "start_count": 904}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 811, "h264_decode_error_count": 1960, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:44:57+09:00", "last_error_summary": "[h264 @ 0x55f359880640] error while decoding MB 4 28, bytestream -9", "last_exit_at": "2026-07-06T06:45:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:45:00+09:00", "pid": 151035, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.473, "name": "ffmpeg", "pid": 151035, "rss_mb": 139.473, "state": "S (sleeping)", "threads": 106}, "restart_count": 903, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 904}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 501.942, "sample_count": 759, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933345, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:44:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.537, "last_frame_pushed_at": "2026-07-06T06:44:58+09:00", "last_frame_read_age_sec": 4.849, "last_frame_read_at": "2026-07-06T06:44:57+09:00", "mjpeg": {"age_sec": 4.521, "exists": true, "mtime": "2026-07-06T06:44:58.027474+09:00", "mtime_epoch": 1783287898.0274737, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176131}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:45:02+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9547.965, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.875, "pushed": 2.875, "read": 2.841}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.66, "count": 85, "max_ms": 43.274, "p95_ms": 38.823}, "gstreamer_push": {"avg_ms": 0.87, "count": 86, "max_ms": 1.097, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.82, "count": 85, "max_ms": 66.748, "p95_ms": 62.782}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.01, "count": 85, "max_ms": 92.937, "p95_ms": 89.997}, "yolo11_640_fire_11class": {"avg_ms": 49.799, "count": 85, "max_ms": 68.279, "p95_ms": 65.135}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.214, "count": 85, "max_ms": 95.771, "p95_ms": 92.172}}, "truncated_count": 0}, "processing_total": {"avg_ms": 289.24, "count": 86, "max_ms": 326.312, "p95_ms": 314.458}, "read_interval": {"avg_ms": 298.412, "count": 85, "max_ms": 335.097, "p95_ms": 323.437}, "triton_wait": {"avg_ms": 72.965, "count": 85, "max_ms": 95.783, "p95_ms": 92.185}, "visualize": {"avg_ms": 10.127, "count": 85, "max_ms": 14.355, "p95_ms": 11.257}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16114}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16434.891, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5560a61f7400] error while decoding MB 14 39, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 812, "h264_decode_error_count": 1961, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:47:04+09:00", "last_error_summary": "[h264 @ 0x5560a61f7400] error while decoding MB 14 39, bytestream -7", "last_exit_at": "2026-07-06T06:45:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.188, "last_started_at": "2026-07-06T06:45:00+09:00", "pid": 151035, "process": {"exists": false, "pid": 151035}, "restart_count": 903, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 904}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 447.498, "sample_count": 854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933723, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:47:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T06:47:04+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-06T06:47:04+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T06:47:04.694207+09:00", "mtime_epoch": 1783288024.694207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182048}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:47:04+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.223, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 86, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.882, "pushed": 2.882, "read": 2.882}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 37.999, "count": 85, "max_ms": 46.336, "p95_ms": 40.797}, "gstreamer_push": {"avg_ms": 0.872, "count": 86, "max_ms": 1.24, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.337, "count": 85, "max_ms": 66.382, "p95_ms": 61.385}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.346, "count": 85, "max_ms": 96.139, "p95_ms": 89.399}, "yolo11_640_fire_11class": {"avg_ms": 49.694, "count": 85, "max_ms": 67.911, "p95_ms": 64.062}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.346, "count": 85, "max_ms": 94.328, "p95_ms": 92.474}}, "truncated_count": 0}, "processing_total": {"avg_ms": 340.875, "count": 86, "max_ms": 382.617, "p95_ms": 375.578}, "read_interval": {"avg_ms": 350.533, "count": 86, "max_ms": 396.079, "p95_ms": 384.527}, "triton_wait": {"avg_ms": 74.269, "count": 85, "max_ms": 96.151, "p95_ms": 93.078}, "visualize": {"avg_ms": 12.08, "count": 85, "max_ms": 17.54, "p95_ms": 13.685}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.839, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16114}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16434.391, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:47:04+09:00", "last_error_summary": "[h264 @ 0x5560a61f7400] error while decoding MB 14 39, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 812, "h264_decode_error_count": 1961, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:47:04+09:00", "last_error_summary": "[h264 @ 0x5560a61f7400] error while decoding MB 14 39, bytestream -7", "last_exit_at": "2026-07-06T06:47:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.435, "last_started_at": "2026-07-06T06:45:00+09:00", "pid": 151035, "process": {"exists": false, "pid": 151035}, "restart_count": 903, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 904}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 447.498, "sample_count": 854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933724, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:47:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:47:04+09:00", "last_frame_read_age_sec": 0.296, "last_frame_read_at": "2026-07-06T06:47:04+09:00", "mjpeg": {"age_sec": 0.288, "exists": true, "mtime": "2026-07-06T06:47:04.694207+09:00", "mtime_epoch": 1783288024.694207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182048}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:47:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.223, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.894, "pushed": 2.894, "read": 2.86}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 37.983, "count": 86, "max_ms": 46.336, "p95_ms": 40.797}, "gstreamer_push": {"avg_ms": 0.872, "count": 86, "max_ms": 1.24, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.379, "count": 86, "max_ms": 66.382, "p95_ms": 61.385}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.386, "count": 86, "max_ms": 96.139, "p95_ms": 89.399}, "yolo11_640_fire_11class": {"avg_ms": 49.723, "count": 86, "max_ms": 67.911, "p95_ms": 64.062}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.34, "count": 86, "max_ms": 94.328, "p95_ms": 92.474}}, "truncated_count": 0}, "processing_total": {"avg_ms": 340.113, "count": 86, "max_ms": 382.617, "p95_ms": 375.578}, "read_interval": {"avg_ms": 350.345, "count": 85, "max_ms": 396.079, "p95_ms": 384.527}, "triton_wait": {"avg_ms": 74.275, "count": 86, "max_ms": 96.151, "p95_ms": 93.078}, "visualize": {"avg_ms": 12.058, "count": 86, "max_ms": 17.54, "p95_ms": 13.685}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.72, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16114}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16434.383, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 812, "h264_decode_error_count": 1961, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:47:04+09:00", "last_error_summary": "[h264 @ 0x5560a61f7400] error while decoding MB 14 39, bytestream -7", "last_exit_at": "2026-07-06T06:47:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.438, "last_started_at": "2026-07-06T06:45:00+09:00", "pid": 151035, "process": {"exists": false, "pid": 151035}, "restart_count": 903, "restart_count_per_hour": 25, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 904}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 447.498, "sample_count": 854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933724, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:47:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:47:04+09:00", "last_frame_read_age_sec": 0.299, "last_frame_read_at": "2026-07-06T06:47:04+09:00", "mjpeg": {"age_sec": 0.291, "exists": true, "mtime": "2026-07-06T06:47:04.694207+09:00", "mtime_epoch": 1783288024.694207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182048}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:47:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.223, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.893, "pushed": 2.893, "read": 2.86}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 37.983, "count": 86, "max_ms": 46.336, "p95_ms": 40.797}, "gstreamer_push": {"avg_ms": 0.872, "count": 86, "max_ms": 1.24, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.379, "count": 86, "max_ms": 66.382, "p95_ms": 61.385}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.386, "count": 86, "max_ms": 96.139, "p95_ms": 89.399}, "yolo11_640_fire_11class": {"avg_ms": 49.723, "count": 86, "max_ms": 67.911, "p95_ms": 64.062}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.34, "count": 86, "max_ms": 94.328, "p95_ms": 92.474}}, "truncated_count": 0}, "processing_total": {"avg_ms": 340.113, "count": 86, "max_ms": 382.617, "p95_ms": 375.578}, "read_interval": {"avg_ms": 350.345, "count": 85, "max_ms": 396.079, "p95_ms": 384.527}, "triton_wait": {"avg_ms": 74.275, "count": 86, "max_ms": 96.151, "p95_ms": 93.078}, "visualize": {"avg_ms": 12.058, "count": 86, "max_ms": 17.54, "p95_ms": 13.685}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.723, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16114}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16543.02, "limit_mb": 8796093022207.996}, "detail": {"pid": 151142, "restart_count": 904, "start_count": 905}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 812, "h264_decode_error_count": 1961, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:47:04+09:00", "last_error_summary": "[h264 @ 0x5560a61f7400] error while decoding MB 14 39, bytestream -7", "last_exit_at": "2026-07-06T06:47:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:47:07+09:00", "pid": 151142, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.895, "name": "ffmpeg", "pid": 151142, "rss_mb": 138.895, "state": "S (sleeping)", "threads": 106}, "restart_count": 904, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 905}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 448.117, "sample_count": 843, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1933724, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:47:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.281, "last_frame_pushed_at": "2026-07-06T06:47:04+09:00", "last_frame_read_age_sec": 4.574, "last_frame_read_at": "2026-07-06T06:47:04+09:00", "mjpeg": {"age_sec": 4.266, "exists": true, "mtime": "2026-07-06T06:47:04.994213+09:00", "mtime_epoch": 1783288024.994213, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181866}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:47:09+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.223, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.48, "pushed": 2.48, "read": 2.447}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 37.999, "count": 73, "max_ms": 46.336, "p95_ms": 40.618}, "gstreamer_push": {"avg_ms": 0.87, "count": 74, "max_ms": 1.24, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.877, "count": 73, "max_ms": 63.043, "p95_ms": 61.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.897, "count": 73, "max_ms": 96.139, "p95_ms": 89.399}, "yolo11_640_fire_11class": {"avg_ms": 49.489, "count": 73, "max_ms": 65.554, "p95_ms": 63.758}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.044, "count": 73, "max_ms": 94.328, "p95_ms": 93.065}}, "truncated_count": 0}, "processing_total": {"avg_ms": 340.268, "count": 74, "max_ms": 382.617, "p95_ms": 376.225}, "read_interval": {"avg_ms": 350.599, "count": 73, "max_ms": 396.079, "p95_ms": 385.284}, "triton_wait": {"avg_ms": 73.975, "count": 73, "max_ms": 96.151, "p95_ms": 93.078}, "visualize": {"avg_ms": 12.094, "count": 73, "max_ms": 17.54, "p95_ms": 14.301}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.837, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16534.531, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[NULL @ 0x564ea3cc8300] missing picture in access unit with size 39 [h264 @ 0x564ea4752740] no frame! [vist#0:0/h264 @ 0x564ea3d4b640] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 813, "h264_decode_error_count": 1962, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:49:08+09:00", "last_error_summary": "[NULL @ 0x564ea3cc8300] missing picture in access unit with size 39 [h264 @ 0x564ea4752740] no frame! [vist#0:0/h264 @ 0x564ea3d4b640] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T06:47:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.716, "last_started_at": "2026-07-06T06:47:07+09:00", "pid": 151142, "process": {"exists": false, "pid": 151142}, "restart_count": 904, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 905}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 371.506, "sample_count": 926, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1934123, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:49:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T06:49:08+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T06:49:08+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T06:49:08.928798+09:00", "mtime_epoch": 1783288148.9287984, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173004}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:49:08+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.48, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.289, "pushed": 3.289, "read": 3.289}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.574, "count": 98, "max_ms": 46.78, "p95_ms": 41.568}, "gstreamer_push": {"avg_ms": 0.864, "count": 98, "max_ms": 1.058, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.402, "count": 98, "max_ms": 71.611, "p95_ms": 64.718}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.272, "count": 98, "max_ms": 95.862, "p95_ms": 92.481}, "yolo11_640_fire_11class": {"avg_ms": 49.921, "count": 98, "max_ms": 67.429, "p95_ms": 65.77}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.744, "count": 98, "max_ms": 93.784, "p95_ms": 90.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 297.499, "count": 98, "max_ms": 377.925, "p95_ms": 353.987}, "read_interval": {"avg_ms": 306.86, "count": 98, "max_ms": 387.655, "p95_ms": 363.791}, "triton_wait": {"avg_ms": 71.457, "count": 98, "max_ms": 95.874, "p95_ms": 92.801}, "visualize": {"avg_ms": 10.185, "count": 98, "max_ms": 13.829, "p95_ms": 11.569}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.794, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16533.734, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:49:08+09:00", "last_error_summary": "[NULL @ 0x564ea3cc8300] missing picture in access unit with size 39 [h264 @ 0x564ea4752740] no frame! [vist#0:0/h264 @ 0x564ea3d4b640] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 813, "h264_decode_error_count": 1962, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:49:08+09:00", "last_error_summary": "[NULL @ 0x564ea3cc8300] missing picture in access unit with size 39 [h264 @ 0x564ea4752740] no frame! [vist#0:0/h264 @ 0x564ea3d4b640] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T06:49:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.008, "last_started_at": "2026-07-06T06:47:07+09:00", "pid": 151142, "process": {"exists": false, "pid": 151142}, "restart_count": 904, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 905}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 371.506, "sample_count": 927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1934124, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:49:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:49:09+09:00", "last_frame_read_age_sec": 0.344, "last_frame_read_at": "2026-07-06T06:49:08+09:00", "mjpeg": {"age_sec": 0.338, "exists": true, "mtime": "2026-07-06T06:49:08.928798+09:00", "mtime_epoch": 1783288148.9287984, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173004}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:49:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.29, "pushed": 3.29, "read": 3.256}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.603, "count": 98, "max_ms": 46.78, "p95_ms": 41.568}, "gstreamer_push": {"avg_ms": 0.864, "count": 98, "max_ms": 1.058, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.432, "count": 98, "max_ms": 71.611, "p95_ms": 64.718}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.272, "count": 98, "max_ms": 95.862, "p95_ms": 92.481}, "yolo11_640_fire_11class": {"avg_ms": 49.963, "count": 98, "max_ms": 67.429, "p95_ms": 65.77}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.74, "count": 98, "max_ms": 93.784, "p95_ms": 89.81}}, "truncated_count": 0}, "processing_total": {"avg_ms": 297.59, "count": 98, "max_ms": 377.925, "p95_ms": 353.987}, "read_interval": {"avg_ms": 306.519, "count": 97, "max_ms": 387.655, "p95_ms": 363.791}, "triton_wait": {"avg_ms": 71.458, "count": 98, "max_ms": 95.874, "p95_ms": 92.826}, "visualize": {"avg_ms": 10.178, "count": 98, "max_ms": 13.829, "p95_ms": 11.569}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.789, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16533.727, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 813, "h264_decode_error_count": 1962, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:49:08+09:00", "last_error_summary": "[NULL @ 0x564ea3cc8300] missing picture in access unit with size 39 [h264 @ 0x564ea4752740] no frame! [vist#0:0/h264 @ 0x564ea3d4b640] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T06:49:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.011, "last_started_at": "2026-07-06T06:47:07+09:00", "pid": 151142, "process": {"exists": false, "pid": 151142}, "restart_count": 904, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 905}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 371.506, "sample_count": 927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1934124, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:49:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T06:49:09+09:00", "last_frame_read_age_sec": 0.348, "last_frame_read_at": "2026-07-06T06:49:08+09:00", "mjpeg": {"age_sec": 0.341, "exists": true, "mtime": "2026-07-06T06:49:08.928798+09:00", "mtime_epoch": 1783288148.9287984, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173004}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:49:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.289, "pushed": 3.289, "read": 3.256}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.603, "count": 98, "max_ms": 46.78, "p95_ms": 41.568}, "gstreamer_push": {"avg_ms": 0.864, "count": 98, "max_ms": 1.058, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.432, "count": 98, "max_ms": 71.611, "p95_ms": 64.718}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.272, "count": 98, "max_ms": 95.862, "p95_ms": 92.481}, "yolo11_640_fire_11class": {"avg_ms": 49.963, "count": 98, "max_ms": 67.429, "p95_ms": 65.77}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.74, "count": 98, "max_ms": 93.784, "p95_ms": 89.81}}, "truncated_count": 0}, "processing_total": {"avg_ms": 297.59, "count": 98, "max_ms": 377.925, "p95_ms": 353.987}, "read_interval": {"avg_ms": 306.519, "count": 97, "max_ms": 387.655, "p95_ms": 363.791}, "triton_wait": {"avg_ms": 71.458, "count": 98, "max_ms": 95.874, "p95_ms": 92.826}, "visualize": {"avg_ms": 10.178, "count": 98, "max_ms": 13.829, "p95_ms": 11.569}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.793, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16633.887, "limit_mb": 8796093022207.996}, "detail": {"pid": 151309, "restart_count": 905, "start_count": 906}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 813, "h264_decode_error_count": 1962, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:49:08+09:00", "last_error_summary": "[NULL @ 0x564ea3cc8300] missing picture in access unit with size 39 [h264 @ 0x564ea4752740] no frame! [vist#0:0/h264 @ 0x564ea3d4b640] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T06:49:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:49:11+09:00", "pid": 151309, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.625, "name": "ffmpeg", "pid": 151309, "rss_mb": 138.625, "state": "S (sleeping)", "threads": 106}, "restart_count": 905, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 906}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 367.478, "sample_count": 916, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1934124, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:49:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.456, "last_frame_pushed_at": "2026-07-06T06:49:09+09:00", "last_frame_read_age_sec": 4.797, "last_frame_read_at": "2026-07-06T06:49:08+09:00", "mjpeg": {"age_sec": 4.439, "exists": true, "mtime": "2026-07-06T06:49:09.280806+09:00", "mtime_epoch": 1783288149.2808056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173533}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:49:13+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.48, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.814, "pushed": 2.814, "read": 2.78}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.709, "count": 83, "max_ms": 46.78, "p95_ms": 41.568}, "gstreamer_push": {"avg_ms": 0.866, "count": 84, "max_ms": 1.058, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.149, "count": 83, "max_ms": 71.611, "p95_ms": 64.673}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.356, "count": 83, "max_ms": 95.862, "p95_ms": 89.159}, "yolo11_640_fire_11class": {"avg_ms": 49.538, "count": 83, "max_ms": 67.429, "p95_ms": 65.603}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.655, "count": 83, "max_ms": 93.784, "p95_ms": 88.779}}, "truncated_count": 0}, "processing_total": {"avg_ms": 296.555, "count": 84, "max_ms": 377.925, "p95_ms": 353.987}, "read_interval": {"avg_ms": 305.366, "count": 83, "max_ms": 387.655, "p95_ms": 363.791}, "triton_wait": {"avg_ms": 70.608, "count": 83, "max_ms": 95.874, "p95_ms": 89.17}, "visualize": {"avg_ms": 10.05, "count": 83, "max_ms": 13.829, "p95_ms": 11.421}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.855, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16121}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16549.648, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c8387ba880] error while decoding MB 19 20, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 814, "h264_decode_error_count": 1963, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:50:59+09:00", "last_error_summary": "[h264 @ 0x55c8387ba880] error while decoding MB 19 20, bytestream -9", "last_exit_at": "2026-07-06T06:49:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.788, "last_started_at": "2026-07-06T06:49:11+09:00", "pid": 151309, "process": {"exists": false, "pid": 151309}, "restart_count": 905, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 906}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 351.407, "sample_count": 1054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1934584, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:50:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T06:50:59+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T06:50:59+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T06:50:59.459045+09:00", "mtime_epoch": 1783288259.4590454, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:50:59+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.48, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.592, "pushed": 4.592, "read": 4.592}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.229, "count": 137, "max_ms": 37.894, "p95_ms": 34.962}, "gstreamer_push": {"avg_ms": 0.87, "count": 137, "max_ms": 0.987, "p95_ms": 0.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.988, "count": 137, "max_ms": 67.704, "p95_ms": 64.115}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.179, "count": 137, "max_ms": 94.99, "p95_ms": 81.394}, "yolo11_640_fire_11class": {"avg_ms": 49.552, "count": 137, "max_ms": 67.831, "p95_ms": 64.498}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.417, "count": 137, "max_ms": 87.303, "p95_ms": 81.812}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.075, "count": 137, "max_ms": 261.596, "p95_ms": 239.976}, "read_interval": {"avg_ms": 218.811, "count": 137, "max_ms": 271.736, "p95_ms": 250.658}, "triton_wait": {"avg_ms": 65.465, "count": 137, "max_ms": 95.002, "p95_ms": 83.957}, "visualize": {"avg_ms": 7.462, "count": 137, "max_ms": 9.474, "p95_ms": 9.12}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.832, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16121}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16554.902, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:50:59+09:00", "last_error_summary": "[h264 @ 0x55c8387ba880] error while decoding MB 19 20, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 814, "h264_decode_error_count": 1963, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:50:59+09:00", "last_error_summary": "[h264 @ 0x55c8387ba880] error while decoding MB 19 20, bytestream -9", "last_exit_at": "2026-07-06T06:50:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.978, "last_started_at": "2026-07-06T06:49:11+09:00", "pid": 151309, "process": {"exists": false, "pid": 151309}, "restart_count": 905, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 906}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 351.407, "sample_count": 1054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1934585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:50:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:50:59+09:00", "last_frame_read_age_sec": 0.244, "last_frame_read_at": "2026-07-06T06:50:59+09:00", "mjpeg": {"age_sec": 0.238, "exists": true, "mtime": "2026-07-06T06:50:59.459045+09:00", "mtime_epoch": 1783288259.4590454, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:50:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.595, "pushed": 4.595, "read": 4.561}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.253, "count": 137, "max_ms": 37.894, "p95_ms": 34.964}, "gstreamer_push": {"avg_ms": 0.87, "count": 137, "max_ms": 0.987, "p95_ms": 0.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.948, "count": 137, "max_ms": 67.704, "p95_ms": 64.115}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.139, "count": 137, "max_ms": 94.99, "p95_ms": 81.394}, "yolo11_640_fire_11class": {"avg_ms": 49.478, "count": 137, "max_ms": 67.831, "p95_ms": 64.498}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.397, "count": 137, "max_ms": 87.303, "p95_ms": 81.812}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.366, "count": 137, "max_ms": 261.596, "p95_ms": 239.976}, "read_interval": {"avg_ms": 218.882, "count": 136, "max_ms": 271.736, "p95_ms": 250.658}, "triton_wait": {"avg_ms": 65.431, "count": 137, "max_ms": 95.002, "p95_ms": 83.957}, "visualize": {"avg_ms": 7.479, "count": 137, "max_ms": 9.474, "p95_ms": 9.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.816, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16121}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16555.02, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 814, "h264_decode_error_count": 1963, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:50:59+09:00", "last_error_summary": "[h264 @ 0x55c8387ba880] error while decoding MB 19 20, bytestream -9", "last_exit_at": "2026-07-06T06:50:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.981, "last_started_at": "2026-07-06T06:49:11+09:00", "pid": 151309, "process": {"exists": false, "pid": 151309}, "restart_count": 905, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 906}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 351.407, "sample_count": 1054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1934585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:50:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T06:50:59+09:00", "last_frame_read_age_sec": 0.248, "last_frame_read_at": "2026-07-06T06:50:59+09:00", "mjpeg": {"age_sec": 0.241, "exists": true, "mtime": "2026-07-06T06:50:59.459045+09:00", "mtime_epoch": 1783288259.4590454, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:50:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.594, "pushed": 4.594, "read": 4.561}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.253, "count": 137, "max_ms": 37.894, "p95_ms": 34.964}, "gstreamer_push": {"avg_ms": 0.87, "count": 137, "max_ms": 0.987, "p95_ms": 0.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.948, "count": 137, "max_ms": 67.704, "p95_ms": 64.115}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.139, "count": 137, "max_ms": 94.99, "p95_ms": 81.394}, "yolo11_640_fire_11class": {"avg_ms": 49.478, "count": 137, "max_ms": 67.831, "p95_ms": 64.498}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.397, "count": 137, "max_ms": 87.303, "p95_ms": 81.812}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.366, "count": 137, "max_ms": 261.596, "p95_ms": 239.976}, "read_interval": {"avg_ms": 218.882, "count": 136, "max_ms": 271.736, "p95_ms": 250.658}, "triton_wait": {"avg_ms": 65.431, "count": 137, "max_ms": 95.002, "p95_ms": 83.957}, "visualize": {"avg_ms": 7.479, "count": 137, "max_ms": 9.474, "p95_ms": 9.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.82, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16121}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16665.68, "limit_mb": 8796093022207.996}, "detail": {"pid": 151476, "restart_count": 906, "start_count": 907}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 814, "h264_decode_error_count": 1963, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:50:59+09:00", "last_error_summary": "[h264 @ 0x55c8387ba880] error while decoding MB 19 20, bytestream -9", "last_exit_at": "2026-07-06T06:51:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:51:02+09:00", "pid": 151476, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.16, "name": "ffmpeg", "pid": 151476, "rss_mb": 139.16, "state": "S (sleeping)", "threads": 106}, "restart_count": 906, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 907}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 352.135, "sample_count": 1040, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1934585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:50:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.685, "last_frame_pushed_at": "2026-07-06T06:50:59+09:00", "last_frame_read_age_sec": 4.926, "last_frame_read_at": "2026-07-06T06:50:59+09:00", "mjpeg": {"age_sec": 4.668, "exists": true, "mtime": "2026-07-06T06:50:59.711051+09:00", "mtime_epoch": 1783288259.7110505, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177560}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:51:04+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.48, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.87, "pushed": 3.87, "read": 3.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.16, "count": 115, "max_ms": 37.894, "p95_ms": 34.964}, "gstreamer_push": {"avg_ms": 0.869, "count": 116, "max_ms": 0.987, "p95_ms": 0.925}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.962, "count": 115, "max_ms": 67.704, "p95_ms": 63.863}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.369, "count": 115, "max_ms": 94.99, "p95_ms": 81.394}, "yolo11_640_fire_11class": {"avg_ms": 49.591, "count": 115, "max_ms": 67.831, "p95_ms": 64.498}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.604, "count": 115, "max_ms": 87.303, "p95_ms": 82.228}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.256, "count": 116, "max_ms": 261.596, "p95_ms": 243.025}, "read_interval": {"avg_ms": 219.72, "count": 115, "max_ms": 271.736, "p95_ms": 253.693}, "triton_wait": {"avg_ms": 65.621, "count": 115, "max_ms": 95.002, "p95_ms": 84.865}, "visualize": {"avg_ms": 7.489, "count": 115, "max_ms": 9.474, "p95_ms": 9.205}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16450.434, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5607427efa40] error while decoding MB 77 21, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 815, "h264_decode_error_count": 1964, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:53:03+09:00", "last_error_summary": "[h264 @ 0x5607427efa40] error while decoding MB 77 21, bytestream -28", "last_exit_at": "2026-07-06T06:51:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.39, "last_started_at": "2026-07-06T06:51:02+09:00", "pid": 151476, "process": {"exists": false, "pid": 151476}, "restart_count": 906, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 907}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 311.376, "sample_count": 1248, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1935153, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:53:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.05, "last_frame_pushed_at": "2026-07-06T06:53:03+09:00", "last_frame_read_age_sec": 0.044, "last_frame_read_at": "2026-07-06T06:53:03+09:00", "mjpeg": {"age_sec": 0.034, "exists": true, "mtime": "2026-07-06T06:53:03.733520+09:00", "mtime_epoch": 1783288383.7335198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:53:03+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.738, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 158, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.302, "pushed": 5.302, "read": 5.302}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.193, "count": 158, "max_ms": 36.09, "p95_ms": 33.925}, "gstreamer_push": {"avg_ms": 0.901, "count": 158, "max_ms": 1.137, "p95_ms": 1.042}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.769, "count": 158, "max_ms": 66.95, "p95_ms": 57.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.018, "count": 158, "max_ms": 89.578, "p95_ms": 64.917}, "yolo11_640_fire_11class": {"avg_ms": 46.031, "count": 158, "max_ms": 65.577, "p95_ms": 51.42}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.414, "count": 158, "max_ms": 87.662, "p95_ms": 77.08}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.825, "count": 158, "max_ms": 237.225, "p95_ms": 220.934}, "read_interval": {"avg_ms": 189.81, "count": 158, "max_ms": 246.911, "p95_ms": 230.958}, "triton_wait": {"avg_ms": 59.626, "count": 158, "max_ms": 89.59, "p95_ms": 77.092}, "visualize": {"avg_ms": 6.696, "count": 158, "max_ms": 9.955, "p95_ms": 8.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.797, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16449.953, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:53:03+09:00", "last_error_summary": "[h264 @ 0x5607427efa40] error while decoding MB 77 21, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 815, "h264_decode_error_count": 1964, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:53:03+09:00", "last_error_summary": "[h264 @ 0x5607427efa40] error while decoding MB 77 21, bytestream -28", "last_exit_at": "2026-07-06T06:53:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.537, "last_started_at": "2026-07-06T06:51:02+09:00", "pid": 151476, "process": {"exists": false, "pid": 151476}, "restart_count": 906, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 907}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 311.376, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1935154, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:53:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:53:03+09:00", "last_frame_read_age_sec": 0.191, "last_frame_read_at": "2026-07-06T06:53:03+09:00", "mjpeg": {"age_sec": 0.181, "exists": true, "mtime": "2026-07-06T06:53:03.733520+09:00", "mtime_epoch": 1783288383.7335198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:53:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.738, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 159, "pushed_frames": 159, "read_frames": 158, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.31, "pushed": 5.31, "read": 5.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.182, "count": 158, "max_ms": 36.09, "p95_ms": 33.925}, "gstreamer_push": {"avg_ms": 0.9, "count": 159, "max_ms": 1.137, "p95_ms": 1.042}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.65, "count": 158, "max_ms": 66.898, "p95_ms": 54.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.855, "count": 158, "max_ms": 89.578, "p95_ms": 64.249}, "yolo11_640_fire_11class": {"avg_ms": 45.917, "count": 158, "max_ms": 62.353, "p95_ms": 51.205}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.233, "count": 158, "max_ms": 87.662, "p95_ms": 70.648}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.865, "count": 159, "max_ms": 237.225, "p95_ms": 220.934}, "read_interval": {"avg_ms": 189.81, "count": 158, "max_ms": 246.911, "p95_ms": 230.958}, "triton_wait": {"avg_ms": 59.445, "count": 158, "max_ms": 89.59, "p95_ms": 70.659}, "visualize": {"avg_ms": 6.687, "count": 158, "max_ms": 9.955, "p95_ms": 8.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16450.07, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 815, "h264_decode_error_count": 1964, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:53:03+09:00", "last_error_summary": "[h264 @ 0x5607427efa40] error while decoding MB 77 21, bytestream -28", "last_exit_at": "2026-07-06T06:53:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.547, "last_started_at": "2026-07-06T06:51:02+09:00", "pid": 151476, "process": {"exists": false, "pid": 151476}, "restart_count": 906, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 907}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 311.376, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1935154, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:53:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T06:53:03+09:00", "last_frame_read_age_sec": 0.201, "last_frame_read_at": "2026-07-06T06:53:03+09:00", "mjpeg": {"age_sec": 0.191, "exists": true, "mtime": "2026-07-06T06:53:03.733520+09:00", "mtime_epoch": 1783288383.7335198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:53:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.738, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 159, "pushed_frames": 159, "read_frames": 158, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.308, "pushed": 5.308, "read": 5.275}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.182, "count": 158, "max_ms": 36.09, "p95_ms": 33.925}, "gstreamer_push": {"avg_ms": 0.9, "count": 159, "max_ms": 1.137, "p95_ms": 1.042}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.65, "count": 158, "max_ms": 66.898, "p95_ms": 54.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.855, "count": 158, "max_ms": 89.578, "p95_ms": 64.249}, "yolo11_640_fire_11class": {"avg_ms": 45.917, "count": 158, "max_ms": 62.353, "p95_ms": 51.205}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.233, "count": 158, "max_ms": 87.662, "p95_ms": 70.648}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.865, "count": 159, "max_ms": 237.225, "p95_ms": 220.934}, "read_interval": {"avg_ms": 189.81, "count": 158, "max_ms": 246.911, "p95_ms": 230.958}, "triton_wait": {"avg_ms": 59.445, "count": 158, "max_ms": 89.59, "p95_ms": 70.659}, "visualize": {"avg_ms": 6.687, "count": 158, "max_ms": 9.955, "p95_ms": 8.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 395, "tracking_summary": 16126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16584.828, "limit_mb": 8796093022207.996}, "detail": {"pid": 151583, "restart_count": 907, "start_count": 908}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 815, "h264_decode_error_count": 1964, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:53:03+09:00", "last_error_summary": "[h264 @ 0x5607427efa40] error while decoding MB 77 21, bytestream -28", "last_exit_at": "2026-07-06T06:53:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:53:06+09:00", "pid": 151583, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.027, "name": "ffmpeg", "pid": 151583, "rss_mb": 142.027, "state": "S (sleeping)", "threads": 106}, "restart_count": 907, "restart_count_per_hour": 28, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 908}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 304.818, "sample_count": 1236, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1935154, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:53:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.419, "last_frame_pushed_at": "2026-07-06T06:53:03+09:00", "last_frame_read_age_sec": 4.606, "last_frame_read_at": "2026-07-06T06:53:03+09:00", "mjpeg": {"age_sec": 4.405, "exists": true, "mtime": "2026-07-06T06:53:03.925524+09:00", "mtime_epoch": 1783288383.9255235, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:53:08+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.738, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.634, "pushed": 4.634, "read": 4.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk"], "latency_ms": {"analyze": {"avg_ms": 29.852, "count": 138, "max_ms": 34.73, "p95_ms": 33.25}, "gstreamer_push": {"avg_ms": 0.901, "count": 139, "max_ms": 1.137, "p95_ms": 1.054}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.535, "count": 138, "max_ms": 65.737, "p95_ms": 48.944}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.737, "count": 138, "max_ms": 89.578, "p95_ms": 62.928}, "yolo11_640_fire_11class": {"avg_ms": 45.78, "count": 138, "max_ms": 62.353, "p95_ms": 48.222}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.834, "count": 138, "max_ms": 87.662, "p95_ms": 62.083}}, "truncated_count": 0}, "processing_total": {"avg_ms": 175.486, "count": 139, "max_ms": 225.659, "p95_ms": 213.683}, "read_interval": {"avg_ms": 185.384, "count": 138, "max_ms": 235.065, "p95_ms": 223.281}, "triton_wait": {"avg_ms": 59.216, "count": 138, "max_ms": 89.59, "p95_ms": 64.261}, "visualize": {"avg_ms": 6.52, "count": 138, "max_ms": 8.394, "p95_ms": 7.656}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 97, "tracking_summary": 20204}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16269.113, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56229d4bfb80] error while decoding MB 28 60, bytestream -15 [in#0/rtsp @ 0x56229c6ede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56229c6ede00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 144, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:53:09+09:00", "last_error_summary": "[h264 @ 0x56229d4bfb80] error while decoding MB 28 60, bytestream -15 [in#0/rtsp @ 0x56229c6ede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56229c6ede00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:25:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 1660.015, "last_started_at": "2026-07-06T06:25:29+09:00", "pid": 149667, "process": {"exists": false, "pid": 149667}, "restart_count": 145, "restart_count_per_hour": 3, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.751, "sample_count": 2582, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2424559, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:53:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T06:53:09+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-06T06:53:09+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-06T06:53:09.529634+09:00", "mtime_epoch": 1783288389.529634, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 140616}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:53:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2765.602, "name": "celery", "pid": 267, "rss_mb": 2598.285, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 45, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.467, "pushed": 1.467, "read": 1.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 12.966, "count": 44, "max_ms": 14.866, "p95_ms": 14.606}, "gstreamer_push": {"avg_ms": 0.978, "count": 44, "max_ms": 1.141, "p95_ms": 1.116}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.951, "count": 44, "max_ms": 63.157, "p95_ms": 60.905}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.929, "count": 44, "max_ms": 88.557, "p95_ms": 78.044}, "yolo11_640_fire_11class": {"avg_ms": 48.453, "count": 44, "max_ms": 61.759, "p95_ms": 57.299}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.545, "count": 44, "max_ms": 85.607, "p95_ms": 82.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.531, "count": 44, "max_ms": 189.26, "p95_ms": 175.115}, "read_interval": {"avg_ms": 668.446, "count": 45, "max_ms": 13460.35, "p95_ms": 198.997}, "triton_wait": {"avg_ms": 59.838, "count": 44, "max_ms": 88.57, "p95_ms": 84.079}, "visualize": {"avg_ms": 5.077, "count": 44, "max_ms": 6.622, "p95_ms": 6.267}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 97, "tracking_summary": 20204}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16268.824, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:53:09+09:00", "last_error_summary": "[h264 @ 0x56229d4bfb80] error while decoding MB 28 60, bytestream -15 [in#0/rtsp @ 0x56229c6ede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56229c6ede00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 144, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:53:09+09:00", "last_error_summary": "[h264 @ 0x56229d4bfb80] error while decoding MB 28 60, bytestream -15 [in#0/rtsp @ 0x56229c6ede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56229c6ede00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:53:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 1660.128, "last_started_at": "2026-07-06T06:25:29+09:00", "pid": 149667, "process": {"exists": false, "pid": 149667}, "restart_count": 145, "restart_count_per_hour": 3, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.801, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2424560, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:53:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T06:53:09+09:00", "last_frame_read_age_sec": 0.181, "last_frame_read_at": "2026-07-06T06:53:09+09:00", "mjpeg": {"age_sec": 0.172, "exists": true, "mtime": "2026-07-06T06:53:09.529634+09:00", "mtime_epoch": 1783288389.529634, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 140616}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:53:09+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2765.602, "name": "celery", "pid": 267, "rss_mb": 2598.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.502, "pushed": 1.502, "read": 1.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.11, "count": 44, "max_ms": 19.591, "p95_ms": 14.65}, "gstreamer_push": {"avg_ms": 0.977, "count": 45, "max_ms": 1.141, "p95_ms": 1.116}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.999, "count": 44, "max_ms": 63.157, "p95_ms": 60.905}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.676, "count": 44, "max_ms": 88.557, "p95_ms": 78.044}, "yolo11_640_fire_11class": {"avg_ms": 48.484, "count": 44, "max_ms": 61.759, "p95_ms": 57.299}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.329, "count": 44, "max_ms": 85.607, "p95_ms": 82.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.124, "count": 45, "max_ms": 189.26, "p95_ms": 175.353}, "read_interval": {"avg_ms": 680.166, "count": 44, "max_ms": 13460.35, "p95_ms": 198.997}, "triton_wait": {"avg_ms": 59.585, "count": 44, "max_ms": 88.57, "p95_ms": 84.079}, "visualize": {"avg_ms": 5.155, "count": 44, "max_ms": 8.397, "p95_ms": 6.493}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 97, "tracking_summary": 20204}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16268.754, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 144, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:53:09+09:00", "last_error_summary": "[h264 @ 0x56229d4bfb80] error while decoding MB 28 60, bytestream -15 [in#0/rtsp @ 0x56229c6ede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56229c6ede00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:53:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 1660.132, "last_started_at": "2026-07-06T06:25:29+09:00", "pid": 149667, "process": {"exists": false, "pid": 149667}, "restart_count": 145, "restart_count_per_hour": 3, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.801, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2424560, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:53:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T06:53:09+09:00", "last_frame_read_age_sec": 0.186, "last_frame_read_at": "2026-07-06T06:53:09+09:00", "mjpeg": {"age_sec": 0.177, "exists": true, "mtime": "2026-07-06T06:53:09.529634+09:00", "mtime_epoch": 1783288389.529634, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 140616}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:53:09+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2765.602, "name": "celery", "pid": 267, "rss_mb": 2598.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.502, "pushed": 1.502, "read": 1.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.11, "count": 44, "max_ms": 19.591, "p95_ms": 14.65}, "gstreamer_push": {"avg_ms": 0.977, "count": 45, "max_ms": 1.141, "p95_ms": 1.116}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.999, "count": 44, "max_ms": 63.157, "p95_ms": 60.905}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.676, "count": 44, "max_ms": 88.557, "p95_ms": 78.044}, "yolo11_640_fire_11class": {"avg_ms": 48.484, "count": 44, "max_ms": 61.759, "p95_ms": 57.299}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.329, "count": 44, "max_ms": 85.607, "p95_ms": 82.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.124, "count": 45, "max_ms": 189.26, "p95_ms": 175.353}, "read_interval": {"avg_ms": 680.166, "count": 44, "max_ms": 13460.35, "p95_ms": 198.997}, "triton_wait": {"avg_ms": 59.585, "count": 44, "max_ms": 88.57, "p95_ms": 84.079}, "visualize": {"avg_ms": 5.155, "count": 44, "max_ms": 8.397, "p95_ms": 6.493}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 97, "tracking_summary": 20204}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16422.293, "limit_mb": 8796093022207.996}, "detail": {"pid": 151690, "restart_count": 146, "start_count": 147}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 144, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:53:09+09:00", "last_error_summary": "[h264 @ 0x56229d4bfb80] error while decoding MB 28 60, bytestream -15 [in#0/rtsp @ 0x56229c6ede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56229c6ede00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:53:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:53:12+09:00", "pid": 151690, "process": {"exists": true, "fd_count": 5, "hwm_mb": 123.43, "name": "ffmpeg", "pid": 151690, "rss_mb": 123.43, "state": "S (sleeping)", "threads": 19}, "restart_count": 146, "restart_count_per_hour": 4, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.533, "sample_count": 2538, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2424560, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:53:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.7, "last_frame_pushed_at": "2026-07-06T06:53:09+09:00", "last_frame_read_age_sec": 4.878, "last_frame_read_at": "2026-07-06T06:53:09+09:00", "mjpeg": {"age_sec": 4.685, "exists": true, "mtime": "2026-07-06T06:53:09.713638+09:00", "mtime_epoch": 1783288389.7136378, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 140764}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T06:53:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2765.602, "name": "celery", "pid": 267, "rss_mb": 2598.285, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 15, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.535, "pushed": 0.535, "read": 0.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.395, "count": 16, "max_ms": 19.591, "p95_ms": 19.591}, "gstreamer_push": {"avg_ms": 0.99, "count": 16, "max_ms": 1.126, "p95_ms": 1.126}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.066, "count": 16, "max_ms": 60.905, "p95_ms": 60.905}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.057, "count": 16, "max_ms": 84.486, "p95_ms": 84.486}, "yolo11_640_fire_11class": {"avg_ms": 49.374, "count": 16, "max_ms": 57.299, "p95_ms": 57.299}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.326, "count": 16, "max_ms": 82.439, "p95_ms": 82.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.891, "count": 16, "max_ms": 176.224, "p95_ms": 176.224}, "read_interval": {"avg_ms": 1678.291, "count": 15, "max_ms": 13460.35, "p95_ms": 13460.35}, "triton_wait": {"avg_ms": 58.685, "count": 16, "max_ms": 84.499, "p95_ms": 84.499}, "visualize": {"avg_ms": 5.119, "count": 16, "max_ms": 8.397, "p95_ms": 8.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16454.465, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55aa2afb2480] error while decoding MB 15 39, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 816, "h264_decode_error_count": 1965, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:55:08+09:00", "last_error_summary": "[h264 @ 0x55aa2afb2480] error while decoding MB 15 39, bytestream -9", "last_exit_at": "2026-07-06T06:53:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.572, "last_started_at": "2026-07-06T06:53:06+09:00", "pid": 151583, "process": {"exists": false, "pid": 151583}, "restart_count": 907, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 908}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.475, "sample_count": 1394, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1935743, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:55:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-06T06:55:08+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T06:55:08+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-06T06:55:08.855967+09:00", "mtime_epoch": 1783288508.8559668, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185512}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:55:08+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.996, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.112, "pushed": 5.112, "read": 5.112}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.167, "count": 152, "max_ms": 36.772, "p95_ms": 32.493}, "gstreamer_push": {"avg_ms": 0.899, "count": 153, "max_ms": 1.167, "p95_ms": 1.037}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.045, "count": 152, "max_ms": 66.89, "p95_ms": 61.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.702, "count": 152, "max_ms": 85.427, "p95_ms": 78.375}, "yolo11_640_fire_11class": {"avg_ms": 49.237, "count": 152, "max_ms": 68.449, "p95_ms": 64.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.573, "count": 152, "max_ms": 82.312, "p95_ms": 78.627}}, "truncated_count": 0}, "processing_total": {"avg_ms": 186.32, "count": 153, "max_ms": 219.409, "p95_ms": 212.711}, "read_interval": {"avg_ms": 196.464, "count": 153, "max_ms": 228.885, "p95_ms": 223.29}, "triton_wait": {"avg_ms": 63.354, "count": 152, "max_ms": 85.44, "p95_ms": 80.712}, "visualize": {"avg_ms": 6.807, "count": 152, "max_ms": 9.455, "p95_ms": 7.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16453.75, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:55:08+09:00", "last_error_summary": "[h264 @ 0x55aa2afb2480] error while decoding MB 15 39, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 816, "h264_decode_error_count": 1965, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:55:08+09:00", "last_error_summary": "[h264 @ 0x55aa2afb2480] error while decoding MB 15 39, bytestream -9", "last_exit_at": "2026-07-06T06:55:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.701, "last_started_at": "2026-07-06T06:53:06+09:00", "pid": 151583, "process": {"exists": false, "pid": 151583}, "restart_count": 907, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 908}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.475, "sample_count": 1395, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1935744, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:55:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T06:55:09+09:00", "last_frame_read_age_sec": 0.185, "last_frame_read_at": "2026-07-06T06:55:08+09:00", "mjpeg": {"age_sec": 0.174, "exists": true, "mtime": "2026-07-06T06:55:08.855967+09:00", "mtime_epoch": 1783288508.8559668, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185512}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:55:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.996, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.124, "pushed": 5.124, "read": 5.091}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.157, "count": 153, "max_ms": 36.772, "p95_ms": 32.493}, "gstreamer_push": {"avg_ms": 0.899, "count": 153, "max_ms": 1.167, "p95_ms": 1.037}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.078, "count": 153, "max_ms": 66.89, "p95_ms": 61.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.742, "count": 153, "max_ms": 85.427, "p95_ms": 78.375}, "yolo11_640_fire_11class": {"avg_ms": 49.257, "count": 153, "max_ms": 68.449, "p95_ms": 64.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.583, "count": 153, "max_ms": 82.312, "p95_ms": 78.627}}, "truncated_count": 0}, "processing_total": {"avg_ms": 186.284, "count": 153, "max_ms": 219.409, "p95_ms": 212.711}, "read_interval": {"avg_ms": 196.47, "count": 152, "max_ms": 228.885, "p95_ms": 223.29}, "triton_wait": {"avg_ms": 63.383, "count": 153, "max_ms": 85.44, "p95_ms": 80.712}, "visualize": {"avg_ms": 6.803, "count": 153, "max_ms": 9.455, "p95_ms": 7.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.858, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16453.742, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 816, "h264_decode_error_count": 1965, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:55:08+09:00", "last_error_summary": "[h264 @ 0x55aa2afb2480] error while decoding MB 15 39, bytestream -9", "last_exit_at": "2026-07-06T06:55:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.705, "last_started_at": "2026-07-06T06:53:06+09:00", "pid": 151583, "process": {"exists": false, "pid": 151583}, "restart_count": 907, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 908}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.475, "sample_count": 1395, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1935744, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:55:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T06:55:09+09:00", "last_frame_read_age_sec": 0.189, "last_frame_read_at": "2026-07-06T06:55:08+09:00", "mjpeg": {"age_sec": 0.178, "exists": true, "mtime": "2026-07-06T06:55:08.855967+09:00", "mtime_epoch": 1783288508.8559668, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185512}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:55:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.996, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.124, "pushed": 5.124, "read": 5.09}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.157, "count": 153, "max_ms": 36.772, "p95_ms": 32.493}, "gstreamer_push": {"avg_ms": 0.899, "count": 153, "max_ms": 1.167, "p95_ms": 1.037}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.078, "count": 153, "max_ms": 66.89, "p95_ms": 61.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.742, "count": 153, "max_ms": 85.427, "p95_ms": 78.375}, "yolo11_640_fire_11class": {"avg_ms": 49.257, "count": 153, "max_ms": 68.449, "p95_ms": 64.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.583, "count": 153, "max_ms": 82.312, "p95_ms": 78.627}}, "truncated_count": 0}, "processing_total": {"avg_ms": 186.284, "count": 153, "max_ms": 219.409, "p95_ms": 212.711}, "read_interval": {"avg_ms": 196.47, "count": 152, "max_ms": 228.885, "p95_ms": 223.29}, "triton_wait": {"avg_ms": 63.383, "count": 153, "max_ms": 85.44, "p95_ms": 80.712}, "visualize": {"avg_ms": 6.803, "count": 153, "max_ms": 9.455, "p95_ms": 7.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16570.164, "limit_mb": 8796093022207.996}, "detail": {"pid": 151902, "restart_count": 908, "start_count": 909}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 816, "h264_decode_error_count": 1965, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:55:08+09:00", "last_error_summary": "[h264 @ 0x55aa2afb2480] error while decoding MB 15 39, bytestream -9", "last_exit_at": "2026-07-06T06:55:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:55:11+09:00", "pid": 151902, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.012, "name": "ffmpeg", "pid": 151902, "rss_mb": 147.012, "state": "S (sleeping)", "threads": 106}, "restart_count": 908, "restart_count_per_hour": 28, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 909}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.696, "sample_count": 1372, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1935744, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:55:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.527, "last_frame_pushed_at": "2026-07-06T06:55:09+09:00", "last_frame_read_age_sec": 4.709, "last_frame_read_at": "2026-07-06T06:55:08+09:00", "mjpeg": {"age_sec": 4.502, "exists": true, "mtime": "2026-07-06T06:55:09.051971+09:00", "mtime_epoch": 1783288509.0519707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180783}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:55:13+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9548.996, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.393, "pushed": 4.393, "read": 4.359}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.135, "count": 131, "max_ms": 36.772, "p95_ms": 33.405}, "gstreamer_push": {"avg_ms": 0.904, "count": 131, "max_ms": 1.167, "p95_ms": 1.039}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.9, "count": 131, "max_ms": 64.289, "p95_ms": 60.444}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.404, "count": 131, "max_ms": 85.427, "p95_ms": 78.375}, "yolo11_640_fire_11class": {"avg_ms": 49.129, "count": 131, "max_ms": 68.449, "p95_ms": 63.888}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.434, "count": 131, "max_ms": 82.312, "p95_ms": 78.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.479, "count": 131, "max_ms": 219.409, "p95_ms": 212.47}, "read_interval": {"avg_ms": 194.675, "count": 130, "max_ms": 228.885, "p95_ms": 222.091}, "triton_wait": {"avg_ms": 63.633, "count": 131, "max_ms": 85.44, "p95_ms": 79.932}, "visualize": {"avg_ms": 6.711, "count": 131, "max_ms": 9.455, "p95_ms": 7.759}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.823, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16456.289, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55602cc14ac0] error while decoding MB 35 31, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 817, "h264_decode_error_count": 1966, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:57:23+09:00", "last_error_summary": "[h264 @ 0x55602cc14ac0] error while decoding MB 35 31, bytestream -5", "last_exit_at": "2026-07-06T06:55:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.842, "last_started_at": "2026-07-06T06:55:11+09:00", "pid": 151902, "process": {"exists": false, "pid": 151902}, "restart_count": 908, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 909}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.071, "sample_count": 1585, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1936527, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T06:57:23+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T06:57:23+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T06:57:23.354559+09:00", "mtime_epoch": 1783288643.354559, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188425}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:57:23+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9549.512, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 181, "read_frames": 181, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.0, "pushed": 6.034, "read": 6.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.112, "count": 180, "max_ms": 46.581, "p95_ms": 38.784}, "gstreamer_push": {"avg_ms": 1.026, "count": 181, "max_ms": 1.855, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.043, "count": 180, "max_ms": 63.174, "p95_ms": 61.464}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.423, "count": 180, "max_ms": 84.499, "p95_ms": 74.114}, "yolo11_640_fire_11class": {"avg_ms": 50.003, "count": 180, "max_ms": 67.841, "p95_ms": 64.474}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.724, "count": 180, "max_ms": 79.322, "p95_ms": 77.013}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.272, "count": 180, "max_ms": 196.425, "p95_ms": 178.578}, "read_interval": {"avg_ms": 166.4, "count": 181, "max_ms": 212.4, "p95_ms": 189.213}, "triton_wait": {"avg_ms": 60.71, "count": 180, "max_ms": 84.514, "p95_ms": 78.414}, "visualize": {"avg_ms": 5.419, "count": 180, "max_ms": 7.178, "p95_ms": 5.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16455.305, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:57:23+09:00", "last_error_summary": "[h264 @ 0x55602cc14ac0] error while decoding MB 35 31, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 817, "h264_decode_error_count": 1966, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:57:23+09:00", "last_error_summary": "[h264 @ 0x55602cc14ac0] error while decoding MB 35 31, bytestream -5", "last_exit_at": "2026-07-06T06:57:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.976, "last_started_at": "2026-07-06T06:55:11+09:00", "pid": 151902, "process": {"exists": false, "pid": 151902}, "restart_count": 908, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 909}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.071, "sample_count": 1586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1936528, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T06:57:23+09:00", "last_frame_read_age_sec": 0.185, "last_frame_read_at": "2026-07-06T06:57:23+09:00", "mjpeg": {"age_sec": 0.174, "exists": true, "mtime": "2026-07-06T06:57:23.354559+09:00", "mtime_epoch": 1783288643.354559, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188425}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:57:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9549.512, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.041, "pushed": 6.041, "read": 6.008}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.118, "count": 180, "max_ms": 46.581, "p95_ms": 38.784}, "gstreamer_push": {"avg_ms": 1.026, "count": 181, "max_ms": 1.855, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.085, "count": 180, "max_ms": 68.008, "p95_ms": 61.637}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.567, "count": 180, "max_ms": 87.546, "p95_ms": 75.146}, "yolo11_640_fire_11class": {"avg_ms": 50.046, "count": 180, "max_ms": 71.189, "p95_ms": 64.559}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.766, "count": 180, "max_ms": 82.037, "p95_ms": 77.059}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.406, "count": 181, "max_ms": 196.425, "p95_ms": 178.684}, "read_interval": {"avg_ms": 166.327, "count": 180, "max_ms": 212.4, "p95_ms": 189.213}, "triton_wait": {"avg_ms": 60.784, "count": 180, "max_ms": 87.558, "p95_ms": 78.428}, "visualize": {"avg_ms": 5.419, "count": 180, "max_ms": 7.178, "p95_ms": 5.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16455.297, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 817, "h264_decode_error_count": 1966, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:57:23+09:00", "last_error_summary": "[h264 @ 0x55602cc14ac0] error while decoding MB 35 31, bytestream -5", "last_exit_at": "2026-07-06T06:57:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.98, "last_started_at": "2026-07-06T06:55:11+09:00", "pid": 151902, "process": {"exists": false, "pid": 151902}, "restart_count": 908, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 909}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.071, "sample_count": 1586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1936528, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T06:57:23+09:00", "last_frame_read_age_sec": 0.189, "last_frame_read_at": "2026-07-06T06:57:23+09:00", "mjpeg": {"age_sec": 0.178, "exists": true, "mtime": "2026-07-06T06:57:23.354559+09:00", "mtime_epoch": 1783288643.354559, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188425}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:57:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9549.512, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.041, "pushed": 6.041, "read": 6.007}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.118, "count": 180, "max_ms": 46.581, "p95_ms": 38.784}, "gstreamer_push": {"avg_ms": 1.026, "count": 181, "max_ms": 1.855, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.085, "count": 180, "max_ms": 68.008, "p95_ms": 61.637}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.567, "count": 180, "max_ms": 87.546, "p95_ms": 75.146}, "yolo11_640_fire_11class": {"avg_ms": 50.046, "count": 180, "max_ms": 71.189, "p95_ms": 64.559}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.766, "count": 180, "max_ms": 82.037, "p95_ms": 77.059}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.406, "count": 181, "max_ms": 196.425, "p95_ms": 178.684}, "read_interval": {"avg_ms": 166.327, "count": 180, "max_ms": 212.4, "p95_ms": 189.213}, "triton_wait": {"avg_ms": 60.784, "count": 180, "max_ms": 87.558, "p95_ms": 78.428}, "visualize": {"avg_ms": 5.419, "count": 180, "max_ms": 7.178, "p95_ms": 5.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16564.816, "limit_mb": 8796093022207.996}, "detail": {"pid": 152009, "restart_count": 909, "start_count": 910}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 817, "h264_decode_error_count": 1966, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:57:23+09:00", "last_error_summary": "[h264 @ 0x55602cc14ac0] error while decoding MB 35 31, bytestream -5", "last_exit_at": "2026-07-06T06:57:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:57:25+09:00", "pid": 152009, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.727, "name": "ffmpeg", "pid": 152009, "rss_mb": 141.727, "state": "S (sleeping)", "threads": 106}, "restart_count": 909, "restart_count_per_hour": 28, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 910}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.089, "sample_count": 1563, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1936528, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.205, "last_frame_pushed_at": "2026-07-06T06:57:23+09:00", "last_frame_read_age_sec": 4.386, "last_frame_read_at": "2026-07-06T06:57:23+09:00", "mjpeg": {"age_sec": 4.188, "exists": true, "mtime": "2026-07-06T06:57:23.542562+09:00", "mtime_epoch": 1783288643.5425625, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186458}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:57:27+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9549.512, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.184, "pushed": 5.184, "read": 5.151}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.355, "count": 155, "max_ms": 46.581, "p95_ms": 44.817}, "gstreamer_push": {"avg_ms": 1.029, "count": 155, "max_ms": 1.855, "p95_ms": 1.213}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.222, "count": 155, "max_ms": 68.008, "p95_ms": 62.126}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.666, "count": 155, "max_ms": 87.546, "p95_ms": 77.532}, "yolo11_640_fire_11class": {"avg_ms": 50.146, "count": 155, "max_ms": 71.189, "p95_ms": 64.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.893, "count": 155, "max_ms": 82.037, "p95_ms": 77.397}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.718, "count": 155, "max_ms": 196.425, "p95_ms": 180.552}, "read_interval": {"avg_ms": 166.637, "count": 154, "max_ms": 212.4, "p95_ms": 194.489}, "triton_wait": {"avg_ms": 60.937, "count": 155, "max_ms": 87.558, "p95_ms": 78.596}, "visualize": {"avg_ms": 5.409, "count": 155, "max_ms": 7.178, "p95_ms": 5.889}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16457.141, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 818, "h264_decode_error_count": 1967, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:59:34+09:00", "last_error_summary": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "last_exit_at": "2026-07-06T06:57:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.518, "last_started_at": "2026-07-06T06:57:25+09:00", "pid": 152009, "process": {"exists": false, "pid": 152009}, "restart_count": 909, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 910}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 194.609, "sample_count": 1676, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1937242, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:59:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T06:59:34+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T06:59:34+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T06:59:34.205051+09:00", "mtime_epoch": 1783288774.2050507, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188156}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:59:34+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9549.77, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.023, "pushed": 6.023, "read": 6.023}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.54, "count": 180, "max_ms": 33.383, "p95_ms": 31.873}, "gstreamer_push": {"avg_ms": 0.988, "count": 180, "max_ms": 1.209, "p95_ms": 1.132}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.757, "count": 180, "max_ms": 69.585, "p95_ms": 63.121}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.745, "count": 180, "max_ms": 84.362, "p95_ms": 75.166}, "yolo11_640_fire_11class": {"avg_ms": 51.132, "count": 180, "max_ms": 67.729, "p95_ms": 65.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.741, "count": 180, "max_ms": 82.874, "p95_ms": 76.785}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.958, "count": 180, "max_ms": 201.634, "p95_ms": 190.281}, "read_interval": {"avg_ms": 166.764, "count": 180, "max_ms": 211.409, "p95_ms": 200.236}, "triton_wait": {"avg_ms": 61.392, "count": 180, "max_ms": 84.374, "p95_ms": 79.303}, "visualize": {"avg_ms": 5.539, "count": 180, "max_ms": 7.079, "p95_ms": 6.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16456.457, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:59:34+09:00", "last_error_summary": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 818, "h264_decode_error_count": 1967, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:59:34+09:00", "last_error_summary": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "last_exit_at": "2026-07-06T06:59:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.668, "last_started_at": "2026-07-06T06:57:25+09:00", "pid": 152009, "process": {"exists": false, "pid": 152009}, "restart_count": 909, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 910}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 194.609, "sample_count": 1676, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1937243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:59:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T06:59:34+09:00", "last_frame_read_age_sec": 0.199, "last_frame_read_at": "2026-07-06T06:59:34+09:00", "mjpeg": {"age_sec": 0.192, "exists": true, "mtime": "2026-07-06T06:59:34.205051+09:00", "mtime_epoch": 1783288774.2050507, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188156}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:59:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9549.77, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.034, "pushed": 6.034, "read": 6.001}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.536, "count": 180, "max_ms": 33.383, "p95_ms": 31.873}, "gstreamer_push": {"avg_ms": 0.988, "count": 180, "max_ms": 1.209, "p95_ms": 1.132}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.819, "count": 180, "max_ms": 69.585, "p95_ms": 63.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.718, "count": 180, "max_ms": 84.362, "p95_ms": 75.166}, "yolo11_640_fire_11class": {"avg_ms": 51.171, "count": 180, "max_ms": 67.729, "p95_ms": 65.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.803, "count": 180, "max_ms": 82.874, "p95_ms": 77.127}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.023, "count": 180, "max_ms": 201.634, "p95_ms": 190.308}, "read_interval": {"avg_ms": 166.62, "count": 179, "max_ms": 211.409, "p95_ms": 200.253}, "triton_wait": {"avg_ms": 61.454, "count": 180, "max_ms": 84.374, "p95_ms": 79.657}, "visualize": {"avg_ms": 5.539, "count": 180, "max_ms": 7.079, "p95_ms": 6.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.829, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16456.449, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 818, "h264_decode_error_count": 1967, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:59:34+09:00", "last_error_summary": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "last_exit_at": "2026-07-06T06:59:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.671, "last_started_at": "2026-07-06T06:57:25+09:00", "pid": 152009, "process": {"exists": false, "pid": 152009}, "restart_count": 909, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 910}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 194.609, "sample_count": 1676, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1937243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:59:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T06:59:34+09:00", "last_frame_read_age_sec": 0.203, "last_frame_read_at": "2026-07-06T06:59:34+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-06T06:59:34.205051+09:00", "mtime_epoch": 1783288774.2050507, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188156}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:59:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9549.77, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.034, "pushed": 6.034, "read": 6.0}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.536, "count": 180, "max_ms": 33.383, "p95_ms": 31.873}, "gstreamer_push": {"avg_ms": 0.988, "count": 180, "max_ms": 1.209, "p95_ms": 1.132}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.819, "count": 180, "max_ms": 69.585, "p95_ms": 63.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.718, "count": 180, "max_ms": 84.362, "p95_ms": 75.166}, "yolo11_640_fire_11class": {"avg_ms": 51.171, "count": 180, "max_ms": 67.729, "p95_ms": 65.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.803, "count": 180, "max_ms": 82.874, "p95_ms": 77.127}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.023, "count": 180, "max_ms": 201.634, "p95_ms": 190.308}, "read_interval": {"avg_ms": 166.62, "count": 179, "max_ms": 211.409, "p95_ms": 200.253}, "triton_wait": {"avg_ms": 61.454, "count": 180, "max_ms": 84.374, "p95_ms": 79.657}, "visualize": {"avg_ms": 5.539, "count": 180, "max_ms": 7.079, "p95_ms": 6.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.832, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16564.562, "limit_mb": 8796093022207.996}, "detail": {"pid": 152116, "restart_count": 910, "start_count": 911}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 818, "h264_decode_error_count": 1967, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:59:34+09:00", "last_error_summary": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "last_exit_at": "2026-07-06T06:59:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T06:59:37+09:00", "pid": 152116, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.508, "name": "ffmpeg", "pid": 152116, "rss_mb": 139.508, "state": "S (sleeping)", "threads": 106}, "restart_count": 910, "restart_count_per_hour": 28, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 911}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 194.392, "sample_count": 1651, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1937243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T06:59:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.801, "last_frame_pushed_at": "2026-07-06T06:59:34+09:00", "last_frame_read_age_sec": 4.995, "last_frame_read_at": "2026-07-06T06:59:34+09:00", "mjpeg": {"age_sec": 4.784, "exists": true, "mtime": "2026-07-06T06:59:34.409055+09:00", "mtime_epoch": 1783288774.4090548, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T06:59:39+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9549.77, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.184, "pushed": 5.184, "read": 5.151}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.466, "count": 155, "max_ms": 33.383, "p95_ms": 32.192}, "gstreamer_push": {"avg_ms": 1.002, "count": 155, "max_ms": 1.209, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.511, "count": 155, "max_ms": 69.585, "p95_ms": 63.992}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.283, "count": 155, "max_ms": 84.362, "p95_ms": 73.017}, "yolo11_640_fire_11class": {"avg_ms": 49.947, "count": 155, "max_ms": 67.201, "p95_ms": 65.187}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.566, "count": 155, "max_ms": 82.874, "p95_ms": 76.785}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.013, "count": 155, "max_ms": 200.426, "p95_ms": 187.039}, "read_interval": {"avg_ms": 162.685, "count": 154, "max_ms": 210.386, "p95_ms": 196.502}, "triton_wait": {"avg_ms": 60.204, "count": 155, "max_ms": 84.374, "p95_ms": 79.303}, "visualize": {"avg_ms": 5.416, "count": 155, "max_ms": 7.079, "p95_ms": 6.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16458.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T06:59:34+09:00", "last_error_summary": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 818, "h264_decode_error_count": 1967, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:59:34+09:00", "last_error_summary": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "last_exit_at": "2026-07-06T07:02:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.931, "last_started_at": "2026-07-06T06:59:37+09:00", "pid": 152116, "process": {"exists": false, "pid": 152116}, "restart_count": 910, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 911}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.844, "sample_count": 1815, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1938328, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:02:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T07:02:21+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-06T07:02:20+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T07:02:21.006718+09:00", "mtime_epoch": 1783288941.0067184, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182703}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:02:21+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9550.285, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.102, "pushed": 6.102, "read": 6.068}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.106, "count": 182, "max_ms": 37.333, "p95_ms": 32.363}, "gstreamer_push": {"avg_ms": 0.994, "count": 183, "max_ms": 1.186, "p95_ms": 1.119}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.984, "count": 182, "max_ms": 61.633, "p95_ms": 59.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.438, "count": 182, "max_ms": 82.49, "p95_ms": 77.427}, "yolo11_640_fire_11class": {"avg_ms": 44.337, "count": 182, "max_ms": 62.936, "p95_ms": 57.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.772, "count": 182, "max_ms": 79.934, "p95_ms": 75.969}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.966, "count": 183, "max_ms": 216.163, "p95_ms": 188.083}, "read_interval": {"avg_ms": 165.152, "count": 182, "max_ms": 225.97, "p95_ms": 199.305}, "triton_wait": {"avg_ms": 60.516, "count": 182, "max_ms": 82.503, "p95_ms": 78.434}, "visualize": {"avg_ms": 5.305, "count": 182, "max_ms": 6.715, "p95_ms": 6.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16458.363, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 818, "h264_decode_error_count": 1967, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:59:34+09:00", "last_error_summary": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "last_exit_at": "2026-07-06T07:02:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.936, "last_started_at": "2026-07-06T06:59:37+09:00", "pid": 152116, "process": {"exists": false, "pid": 152116}, "restart_count": 910, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 911}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.844, "sample_count": 1815, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1938328, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:02:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T07:02:21+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T07:02:20+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T07:02:21.006718+09:00", "mtime_epoch": 1783288941.0067184, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182703}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:02:21+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9550.285, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.101, "pushed": 6.101, "read": 6.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.106, "count": 182, "max_ms": 37.333, "p95_ms": 32.363}, "gstreamer_push": {"avg_ms": 0.994, "count": 183, "max_ms": 1.186, "p95_ms": 1.119}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.984, "count": 182, "max_ms": 61.633, "p95_ms": 59.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.438, "count": 182, "max_ms": 82.49, "p95_ms": 77.427}, "yolo11_640_fire_11class": {"avg_ms": 44.337, "count": 182, "max_ms": 62.936, "p95_ms": 57.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.772, "count": 182, "max_ms": 79.934, "p95_ms": 75.969}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.966, "count": 183, "max_ms": 216.163, "p95_ms": 188.083}, "read_interval": {"avg_ms": 165.152, "count": 182, "max_ms": 225.97, "p95_ms": 199.305}, "triton_wait": {"avg_ms": 60.516, "count": 182, "max_ms": 82.503, "p95_ms": 78.434}, "visualize": {"avg_ms": 5.305, "count": 182, "max_ms": 6.715, "p95_ms": 6.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16566.156, "limit_mb": 8796093022207.996}, "detail": {"pid": 152223, "restart_count": 911, "start_count": 912}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 818, "h264_decode_error_count": 1967, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T06:59:34+09:00", "last_error_summary": "[h264 @ 0x55fc0e59c800] error while decoding MB 22 25, bytestream -10", "last_exit_at": "2026-07-06T07:02:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:02:24+09:00", "pid": 152223, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.316, "name": "ffmpeg", "pid": 152223, "rss_mb": 139.316, "state": "S (sleeping)", "threads": 106}, "restart_count": 911, "restart_count_per_hour": 28, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 912}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.942, "sample_count": 1800, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1938328, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:02:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.894, "last_frame_pushed_at": "2026-07-06T07:02:21+09:00", "last_frame_read_age_sec": 5.015, "last_frame_read_at": "2026-07-06T07:02:20+09:00", "mjpeg": {"age_sec": 4.877, "exists": true, "mtime": "2026-07-06T07:02:21.134724+09:00", "mtime_epoch": 1783288941.1347244, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182938}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:02:26+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9550.285, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.207, "pushed": 5.207, "read": 5.174}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.926, "count": 155, "max_ms": 37.333, "p95_ms": 32.112}, "gstreamer_push": {"avg_ms": 1.008, "count": 156, "max_ms": 1.186, "p95_ms": 1.128}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.896, "count": 155, "max_ms": 61.633, "p95_ms": 59.891}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.451, "count": 155, "max_ms": 82.49, "p95_ms": 77.427}, "yolo11_640_fire_11class": {"avg_ms": 43.777, "count": 155, "max_ms": 62.936, "p95_ms": 54.894}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.509, "count": 155, "max_ms": 78.42, "p95_ms": 75.681}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.645, "count": 156, "max_ms": 188.019, "p95_ms": 177.977}, "read_interval": {"avg_ms": 161.907, "count": 155, "max_ms": 199.305, "p95_ms": 189.019}, "triton_wait": {"avg_ms": 60.521, "count": 155, "max_ms": 82.503, "p95_ms": 77.854}, "visualize": {"avg_ms": 5.218, "count": 155, "max_ms": 6.715, "p95_ms": 5.725}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16459.887, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561a73aa6100] error while decoding MB 53 22, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 819, "h264_decode_error_count": 1968, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:05:23+09:00", "last_error_summary": "[h264 @ 0x561a73aa6100] error while decoding MB 53 22, bytestream -5", "last_exit_at": "2026-07-06T07:02:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 179.598, "last_started_at": "2026-07-06T07:02:24+09:00", "pid": 152223, "process": {"exists": false, "pid": 152223}, "restart_count": 911, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 912}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.359, "sample_count": 2018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1939579, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:05:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T07:05:23+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T07:05:23+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-06T07:05:23.562520+09:00", "mtime_epoch": 1783289123.5625203, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:05:23+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9551.059, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.173, "pushed": 7.173, "read": 7.173}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.462, "count": 214, "max_ms": 34.591, "p95_ms": 29.819}, "gstreamer_push": {"avg_ms": 1.102, "count": 215, "max_ms": 1.841, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.699, "count": 214, "max_ms": 66.615, "p95_ms": 61.561}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.103, "count": 214, "max_ms": 88.686, "p95_ms": 75.43}, "yolo11_640_fire_11class": {"avg_ms": 48.878, "count": 214, "max_ms": 66.964, "p95_ms": 61.469}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.028, "count": 214, "max_ms": 78.12, "p95_ms": 75.284}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.735, "count": 215, "max_ms": 158.595, "p95_ms": 144.955}, "read_interval": {"avg_ms": 139.727, "count": 215, "max_ms": 170.819, "p95_ms": 157.412}, "triton_wait": {"avg_ms": 60.371, "count": 214, "max_ms": 88.699, "p95_ms": 75.94}, "visualize": {"avg_ms": 4.392, "count": 214, "max_ms": 5.06, "p95_ms": 4.583}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16459.906, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:05:23+09:00", "last_error_summary": "[h264 @ 0x561a73aa6100] error while decoding MB 53 22, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 819, "h264_decode_error_count": 1968, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:05:23+09:00", "last_error_summary": "[h264 @ 0x561a73aa6100] error while decoding MB 53 22, bytestream -5", "last_exit_at": "2026-07-06T07:05:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 179.682, "last_started_at": "2026-07-06T07:02:24+09:00", "pid": 152223, "process": {"exists": false, "pid": 152223}, "restart_count": 911, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 912}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.359, "sample_count": 2018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1939580, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:05:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T07:05:23+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-06T07:05:23+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T07:05:23.562520+09:00", "mtime_epoch": 1783289123.5625203, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:05:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9551.059, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.188, "pushed": 7.188, "read": 7.155}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.472, "count": 215, "max_ms": 34.591, "p95_ms": 29.819}, "gstreamer_push": {"avg_ms": 1.102, "count": 215, "max_ms": 1.841, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.683, "count": 215, "max_ms": 66.615, "p95_ms": 61.561}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.094, "count": 215, "max_ms": 88.686, "p95_ms": 75.43}, "yolo11_640_fire_11class": {"avg_ms": 48.862, "count": 215, "max_ms": 66.964, "p95_ms": 61.469}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.0, "count": 215, "max_ms": 78.12, "p95_ms": 75.284}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.771, "count": 215, "max_ms": 158.595, "p95_ms": 144.955}, "read_interval": {"avg_ms": 139.754, "count": 214, "max_ms": 170.819, "p95_ms": 157.412}, "triton_wait": {"avg_ms": 60.356, "count": 215, "max_ms": 88.699, "p95_ms": 75.94}, "visualize": {"avg_ms": 4.395, "count": 215, "max_ms": 5.184, "p95_ms": 4.59}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16459.758, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 819, "h264_decode_error_count": 1968, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:05:23+09:00", "last_error_summary": "[h264 @ 0x561a73aa6100] error while decoding MB 53 22, bytestream -5", "last_exit_at": "2026-07-06T07:05:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 179.688, "last_started_at": "2026-07-06T07:02:24+09:00", "pid": 152223, "process": {"exists": false, "pid": 152223}, "restart_count": 911, "restart_count_per_hour": 26, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 912}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.359, "sample_count": 2018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1939580, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:05:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T07:05:23+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-06T07:05:23+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-06T07:05:23.562520+09:00", "mtime_epoch": 1783289123.5625203, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:05:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9551.059, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.187, "pushed": 7.187, "read": 7.154}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.472, "count": 215, "max_ms": 34.591, "p95_ms": 29.819}, "gstreamer_push": {"avg_ms": 1.102, "count": 215, "max_ms": 1.841, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.683, "count": 215, "max_ms": 66.615, "p95_ms": 61.561}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.094, "count": 215, "max_ms": 88.686, "p95_ms": 75.43}, "yolo11_640_fire_11class": {"avg_ms": 48.862, "count": 215, "max_ms": 66.964, "p95_ms": 61.469}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.0, "count": 215, "max_ms": 78.12, "p95_ms": 75.284}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.771, "count": 215, "max_ms": 158.595, "p95_ms": 144.955}, "read_interval": {"avg_ms": 139.754, "count": 214, "max_ms": 170.819, "p95_ms": 157.412}, "triton_wait": {"avg_ms": 60.356, "count": 215, "max_ms": 88.699, "p95_ms": 75.94}, "visualize": {"avg_ms": 4.395, "count": 215, "max_ms": 5.184, "p95_ms": 4.59}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16567.707, "limit_mb": 8796093022207.996}, "detail": {"pid": 152330, "restart_count": 912, "start_count": 913}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 819, "h264_decode_error_count": 1968, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:05:23+09:00", "last_error_summary": "[h264 @ 0x561a73aa6100] error while decoding MB 53 22, bytestream -5", "last_exit_at": "2026-07-06T07:05:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:05:26+09:00", "pid": 152330, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.664, "name": "ffmpeg", "pid": 152330, "rss_mb": 139.664, "state": "S (sleeping)", "threads": 106}, "restart_count": 912, "restart_count_per_hour": 27, "rtsp_timeout_count": 498, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 913}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.604, "sample_count": 1986, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1939580, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:05:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.528, "last_frame_pushed_at": "2026-07-06T07:05:23+09:00", "last_frame_read_age_sec": 4.66, "last_frame_read_at": "2026-07-06T07:05:23+09:00", "mjpeg": {"age_sec": 4.51, "exists": true, "mtime": "2026-07-06T07:05:23.706526+09:00", "mtime_epoch": 1783289123.7065258, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179342}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:05:28+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9551.059, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 182, "pushed_frames": 182, "read_frames": 181, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.089, "pushed": 6.089, "read": 6.055}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.571, "count": 182, "max_ms": 34.591, "p95_ms": 29.819}, "gstreamer_push": {"avg_ms": 1.101, "count": 182, "max_ms": 1.841, "p95_ms": 1.162}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.684, "count": 182, "max_ms": 66.615, "p95_ms": 61.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.057, "count": 182, "max_ms": 88.686, "p95_ms": 74.77}, "yolo11_640_fire_11class": {"avg_ms": 48.86, "count": 182, "max_ms": 66.964, "p95_ms": 61.435}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.089, "count": 182, "max_ms": 78.12, "p95_ms": 74.692}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.886, "count": 182, "max_ms": 158.595, "p95_ms": 143.938}, "read_interval": {"avg_ms": 140.077, "count": 181, "max_ms": 170.819, "p95_ms": 156.942}, "triton_wait": {"avg_ms": 60.425, "count": 182, "max_ms": 88.699, "p95_ms": 75.741}, "visualize": {"avg_ms": 4.392, "count": 182, "max_ms": 5.184, "p95_ms": 4.58}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16173}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16462.77, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5629d3b6ccc0] cabac decode of qscale diff failed at 63 42 [h264 @ 0x5629d3b6ccc0] error while decoding MB 63 42, bytestream 51385 [h264 @ 0x5629d3d0a900] cabac decode of qscale diff failed at 38 12 [h264 @ 0x5629d3d0a900] error while decoding MB 38 12, bytestream 358 [in#0/rtsp @ 0x5629d3003e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629d3003e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 820, "h264_decode_error_count": 1972, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:08:37+09:00", "last_error_summary": "[h264 @ 0x5629d3b6ccc0] cabac decode of qscale diff failed at 63 42 [h264 @ 0x5629d3b6ccc0] error while decoding MB 63 42, bytestream 51385 [h264 @ 0x5629d3d0a900] cabac decode of qscale diff failed at 38 12 [h264 @ 0x5629d3d0a900] error while decoding MB 38 12, bytestream 358 [in#0/rtsp @ 0x5629d3003e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629d3003e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:05:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.394, "last_started_at": "2026-07-06T07:05:26+09:00", "pid": 152330, "process": {"exists": false, "pid": 152330}, "restart_count": 912, "restart_count_per_hour": 26, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 913}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.374, "sample_count": 1947, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1940770, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:08:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T07:08:37+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T07:08:37+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-06T07:08:37.557570+09:00", "mtime_epoch": 1783289317.5575695, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183345}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:08:37+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9551.574, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 67, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.239, "pushed": 2.239, "read": 2.239}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.206, "count": 67, "max_ms": 45.393, "p95_ms": 35.501}, "gstreamer_push": {"avg_ms": 1.044, "count": 67, "max_ms": 1.297, "p95_ms": 1.23}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.103, "count": 67, "max_ms": 68.329, "p95_ms": 61.397}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.953, "count": 67, "max_ms": 84.425, "p95_ms": 76.822}, "yolo11_640_fire_11class": {"avg_ms": 48.774, "count": 67, "max_ms": 68.14, "p95_ms": 62.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.427, "count": 67, "max_ms": 81.087, "p95_ms": 75.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.87, "count": 67, "max_ms": 167.667, "p95_ms": 149.977}, "read_interval": {"avg_ms": 447.276, "count": 67, "max_ms": 20621.043, "p95_ms": 169.157}, "triton_wait": {"avg_ms": 59.568, "count": 67, "max_ms": 84.44, "p95_ms": 78.491}, "visualize": {"avg_ms": 4.36, "count": 67, "max_ms": 6.993, "p95_ms": 5.753}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16173}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16462.59, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:08:37+09:00", "last_error_summary": "[h264 @ 0x5629d3b6ccc0] cabac decode of qscale diff failed at 63 42 [h264 @ 0x5629d3b6ccc0] error while decoding MB 63 42, bytestream 51385 [h264 @ 0x5629d3d0a900] cabac decode of qscale diff failed at 38 12 [h264 @ 0x5629d3d0a900] error while decoding MB 38 12, bytestream 358 [in#0/rtsp @ 0x5629d3003e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629d3003e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 820, "h264_decode_error_count": 1972, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:08:37+09:00", "last_error_summary": "[h264 @ 0x5629d3b6ccc0] cabac decode of qscale diff failed at 63 42 [h264 @ 0x5629d3b6ccc0] error while decoding MB 63 42, bytestream 51385 [h264 @ 0x5629d3d0a900] cabac decode of qscale diff failed at 38 12 [h264 @ 0x5629d3d0a900] error while decoding MB 38 12, bytestream 358 [in#0/rtsp @ 0x5629d3003e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629d3003e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:08:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.485, "last_started_at": "2026-07-06T07:05:26+09:00", "pid": 152330, "process": {"exists": false, "pid": 152330}, "restart_count": 912, "restart_count_per_hour": 26, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 913}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.374, "sample_count": 1948, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1940771, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:08:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T07:08:37+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-06T07:08:37+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-06T07:08:37.557570+09:00", "mtime_epoch": 1783289317.5575695, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183345}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:08:37+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9551.574, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.267, "pushed": 2.267, "read": 2.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.203, "count": 67, "max_ms": 45.393, "p95_ms": 35.501}, "gstreamer_push": {"avg_ms": 1.045, "count": 68, "max_ms": 1.297, "p95_ms": 1.23}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.098, "count": 67, "max_ms": 68.329, "p95_ms": 61.397}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.995, "count": 67, "max_ms": 84.425, "p95_ms": 76.822}, "yolo11_640_fire_11class": {"avg_ms": 48.767, "count": 67, "max_ms": 68.14, "p95_ms": 62.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.421, "count": 67, "max_ms": 81.087, "p95_ms": 75.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.064, "count": 68, "max_ms": 167.667, "p95_ms": 149.977}, "read_interval": {"avg_ms": 447.276, "count": 67, "max_ms": 20621.043, "p95_ms": 169.157}, "triton_wait": {"avg_ms": 59.61, "count": 67, "max_ms": 84.44, "p95_ms": 78.491}, "visualize": {"avg_ms": 4.405, "count": 67, "max_ms": 7.173, "p95_ms": 6.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16173}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16462.582, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 820, "h264_decode_error_count": 1972, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:08:37+09:00", "last_error_summary": "[h264 @ 0x5629d3b6ccc0] cabac decode of qscale diff failed at 63 42 [h264 @ 0x5629d3b6ccc0] error while decoding MB 63 42, bytestream 51385 [h264 @ 0x5629d3d0a900] cabac decode of qscale diff failed at 38 12 [h264 @ 0x5629d3d0a900] error while decoding MB 38 12, bytestream 358 [in#0/rtsp @ 0x5629d3003e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629d3003e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:08:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.49, "last_started_at": "2026-07-06T07:05:26+09:00", "pid": 152330, "process": {"exists": false, "pid": 152330}, "restart_count": 912, "restart_count_per_hour": 26, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 913}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.374, "sample_count": 1948, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1940771, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:08:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T07:08:37+09:00", "last_frame_read_age_sec": 0.153, "last_frame_read_at": "2026-07-06T07:08:37+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-06T07:08:37.557570+09:00", "mtime_epoch": 1783289317.5575695, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183345}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:08:37+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9551.574, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.267, "pushed": 2.267, "read": 2.233}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.203, "count": 67, "max_ms": 45.393, "p95_ms": 35.501}, "gstreamer_push": {"avg_ms": 1.045, "count": 68, "max_ms": 1.297, "p95_ms": 1.23}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.098, "count": 67, "max_ms": 68.329, "p95_ms": 61.397}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.995, "count": 67, "max_ms": 84.425, "p95_ms": 76.822}, "yolo11_640_fire_11class": {"avg_ms": 48.767, "count": 67, "max_ms": 68.14, "p95_ms": 62.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.421, "count": 67, "max_ms": 81.087, "p95_ms": 75.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.064, "count": 68, "max_ms": 167.667, "p95_ms": 149.977}, "read_interval": {"avg_ms": 447.276, "count": 67, "max_ms": 20621.043, "p95_ms": 169.157}, "triton_wait": {"avg_ms": 59.61, "count": 67, "max_ms": 84.44, "p95_ms": 78.491}, "visualize": {"avg_ms": 4.405, "count": 67, "max_ms": 7.173, "p95_ms": 6.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 396, "tracking_summary": 16173}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16570.789, "limit_mb": 8796093022207.996}, "detail": {"pid": 152437, "restart_count": 913, "start_count": 914}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 820, "h264_decode_error_count": 1972, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:08:37+09:00", "last_error_summary": "[h264 @ 0x5629d3b6ccc0] cabac decode of qscale diff failed at 63 42 [h264 @ 0x5629d3b6ccc0] error while decoding MB 63 42, bytestream 51385 [h264 @ 0x5629d3d0a900] cabac decode of qscale diff failed at 38 12 [h264 @ 0x5629d3d0a900] error while decoding MB 38 12, bytestream 358 [in#0/rtsp @ 0x5629d3003e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629d3003e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:08:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:08:40+09:00", "pid": 152437, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.262, "name": "ffmpeg", "pid": 152437, "rss_mb": 139.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 913, "restart_count_per_hour": 27, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 914}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.37, "sample_count": 1917, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1940771, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:08:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.309, "last_frame_pushed_at": "2026-07-06T07:08:37+09:00", "last_frame_read_age_sec": 4.45, "last_frame_read_at": "2026-07-06T07:08:37+09:00", "mjpeg": {"age_sec": 4.293, "exists": true, "mtime": "2026-07-06T07:08:37.709574+09:00", "mtime_epoch": 1783289317.7095745, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177795}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:08:42+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9551.574, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 37, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.27, "pushed": 1.27, "read": 1.237}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.481, "count": 38, "max_ms": 45.393, "p95_ms": 45.293}, "gstreamer_push": {"avg_ms": 1.034, "count": 38, "max_ms": 1.297, "p95_ms": 1.266}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.671, "count": 38, "max_ms": 62.118, "p95_ms": 61.397}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.523, "count": 38, "max_ms": 84.425, "p95_ms": 74.271}, "yolo11_640_fire_11class": {"avg_ms": 48.379, "count": 38, "max_ms": 68.14, "p95_ms": 62.628}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.179, "count": 38, "max_ms": 78.561, "p95_ms": 72.701}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.914, "count": 38, "max_ms": 167.667, "p95_ms": 149.977}, "read_interval": {"avg_ms": 691.437, "count": 37, "max_ms": 20621.043, "p95_ms": 182.992}, "triton_wait": {"avg_ms": 59.373, "count": 38, "max_ms": 84.44, "p95_ms": 74.28}, "visualize": {"avg_ms": 4.314, "count": 38, "max_ms": 7.173, "p95_ms": 4.94}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16207}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16474.332, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55aadf566f80] error while decoding MB 39 13, bytestream -5 [h264 @ 0x55aadf631400] cabac decode of qscale diff failed at 76 42 [h264 @ 0x55aadf631400] error while decoding MB 76 42, bytestream 455 [h264 @ 0x55aadf513800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aadf513800] error while decoding MB 0 40, bytestream 6110 [h264 @ 0x55aadf631400] error while decoding MB 21 19, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 821, "h264_decode_error_count": 1977, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:16:42+09:00", "last_error_summary": "[h264 @ 0x55aadf566f80] error while decoding MB 39 13, bytestream -5 [h264 @ 0x55aadf631400] cabac decode of qscale diff failed at 76 42 [h264 @ 0x55aadf631400] error while decoding MB 76 42, bytestream 455 [h264 @ 0x55aadf513800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aadf513800] error while decoding MB 0 40, bytestream 6110 [h264 @ 0x55aadf631400] error while decoding MB 21 19, bytestream -7", "last_exit_at": "2026-07-06T07:08:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 482.242, "last_started_at": "2026-07-06T07:08:40+09:00", "pid": 152437, "process": {"exists": false, "pid": 152437}, "restart_count": 913, "restart_count_per_hour": 23, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 914}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.426, "sample_count": 2686, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1944953, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:16:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.087, "last_frame_pushed_at": "2026-07-06T07:16:42+09:00", "last_frame_read_age_sec": 0.075, "last_frame_read_at": "2026-07-06T07:16:42+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-06T07:16:42.175905+09:00", "mtime_epoch": 1783289802.1759055, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181711}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:16:42+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9553.637, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.859, "pushed": 6.859, "read": 6.859}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.559, "count": 205, "max_ms": 42.068, "p95_ms": 30.567}, "gstreamer_push": {"avg_ms": 1.076, "count": 205, "max_ms": 1.204, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.574, "count": 205, "max_ms": 71.101, "p95_ms": 62.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.13, "count": 205, "max_ms": 86.243, "p95_ms": 78.679}, "yolo11_640_fire_11class": {"avg_ms": 49.021, "count": 205, "max_ms": 67.696, "p95_ms": 62.099}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.099, "count": 205, "max_ms": 81.539, "p95_ms": 78.727}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.709, "count": 205, "max_ms": 187.812, "p95_ms": 157.742}, "read_interval": {"avg_ms": 146.287, "count": 205, "max_ms": 200.454, "p95_ms": 168.99}, "triton_wait": {"avg_ms": 60.983, "count": 205, "max_ms": 86.255, "p95_ms": 79.508}, "visualize": {"avg_ms": 4.517, "count": 205, "max_ms": 9.559, "p95_ms": 5.253}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16207}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16474.086, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:16:42+09:00", "last_error_summary": "[h264 @ 0x55aadf566f80] error while decoding MB 39 13, bytestream -5 [h264 @ 0x55aadf631400] cabac decode of qscale diff failed at 76 42 [h264 @ 0x55aadf631400] error while decoding MB 76 42, bytestream 455 [h264 @ 0x55aadf513800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aadf513800] error while decoding MB 0 40, bytestream 6110 [h264 @ 0x55aadf631400] error while decoding MB 21 19, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 821, "h264_decode_error_count": 1977, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:16:42+09:00", "last_error_summary": "[h264 @ 0x55aadf566f80] error while decoding MB 39 13, bytestream -5 [h264 @ 0x55aadf631400] cabac decode of qscale diff failed at 76 42 [h264 @ 0x55aadf631400] error while decoding MB 76 42, bytestream 455 [h264 @ 0x55aadf513800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aadf513800] error while decoding MB 0 40, bytestream 6110 [h264 @ 0x55aadf631400] error while decoding MB 21 19, bytestream -7", "last_exit_at": "2026-07-06T07:16:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 482.307, "last_started_at": "2026-07-06T07:08:40+09:00", "pid": 152437, "process": {"exists": false, "pid": 152437}, "restart_count": 913, "restart_count_per_hour": 23, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 914}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.515, "sample_count": 2686, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1944954, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:16:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T07:16:42+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-06T07:16:42+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-06T07:16:42.175905+09:00", "mtime_epoch": 1783289802.1759055, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181711}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:16:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9553.637, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.878, "pushed": 6.878, "read": 6.844}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.561, "count": 205, "max_ms": 42.068, "p95_ms": 30.612}, "gstreamer_push": {"avg_ms": 1.076, "count": 206, "max_ms": 1.204, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.563, "count": 205, "max_ms": 71.101, "p95_ms": 62.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.105, "count": 205, "max_ms": 86.243, "p95_ms": 78.679}, "yolo11_640_fire_11class": {"avg_ms": 49.013, "count": 205, "max_ms": 67.696, "p95_ms": 62.099}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.066, "count": 205, "max_ms": 81.539, "p95_ms": 78.727}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.699, "count": 206, "max_ms": 187.812, "p95_ms": 157.742}, "read_interval": {"avg_ms": 146.287, "count": 205, "max_ms": 200.454, "p95_ms": 168.99}, "triton_wait": {"avg_ms": 60.958, "count": 205, "max_ms": 86.255, "p95_ms": 79.508}, "visualize": {"avg_ms": 4.504, "count": 205, "max_ms": 9.353, "p95_ms": 5.253}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16207}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16474.078, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 821, "h264_decode_error_count": 1977, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:16:42+09:00", "last_error_summary": "[h264 @ 0x55aadf566f80] error while decoding MB 39 13, bytestream -5 [h264 @ 0x55aadf631400] cabac decode of qscale diff failed at 76 42 [h264 @ 0x55aadf631400] error while decoding MB 76 42, bytestream 455 [h264 @ 0x55aadf513800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aadf513800] error while decoding MB 0 40, bytestream 6110 [h264 @ 0x55aadf631400] error while decoding MB 21 19, bytestream -7", "last_exit_at": "2026-07-06T07:16:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 482.314, "last_started_at": "2026-07-06T07:08:40+09:00", "pid": 152437, "process": {"exists": false, "pid": 152437}, "restart_count": 913, "restart_count_per_hour": 23, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 914}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.515, "sample_count": 2686, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1944954, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:16:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T07:16:42+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-06T07:16:42+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-06T07:16:42.175905+09:00", "mtime_epoch": 1783289802.1759055, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181711}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:16:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9553.637, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.876, "pushed": 6.876, "read": 6.843}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.561, "count": 205, "max_ms": 42.068, "p95_ms": 30.612}, "gstreamer_push": {"avg_ms": 1.076, "count": 206, "max_ms": 1.204, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.563, "count": 205, "max_ms": 71.101, "p95_ms": 62.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.105, "count": 205, "max_ms": 86.243, "p95_ms": 78.679}, "yolo11_640_fire_11class": {"avg_ms": 49.013, "count": 205, "max_ms": 67.696, "p95_ms": 62.099}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.066, "count": 205, "max_ms": 81.539, "p95_ms": 78.727}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.699, "count": 206, "max_ms": 187.812, "p95_ms": 157.742}, "read_interval": {"avg_ms": 146.287, "count": 205, "max_ms": 200.454, "p95_ms": 168.99}, "triton_wait": {"avg_ms": 60.958, "count": 205, "max_ms": 86.255, "p95_ms": 79.508}, "visualize": {"avg_ms": 4.504, "count": 205, "max_ms": 9.353, "p95_ms": 5.253}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16207}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16585.066, "limit_mb": 8796093022207.996}, "detail": {"pid": 152604, "restart_count": 914, "start_count": 915}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 821, "h264_decode_error_count": 1977, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:16:42+09:00", "last_error_summary": "[h264 @ 0x55aadf566f80] error while decoding MB 39 13, bytestream -5 [h264 @ 0x55aadf631400] cabac decode of qscale diff failed at 76 42 [h264 @ 0x55aadf631400] error while decoding MB 76 42, bytestream 455 [h264 @ 0x55aadf513800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aadf513800] error while decoding MB 0 40, bytestream 6110 [h264 @ 0x55aadf631400] error while decoding MB 21 19, bytestream -7", "last_exit_at": "2026-07-06T07:16:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:16:44+09:00", "pid": 152604, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.512, "name": "ffmpeg", "pid": 152604, "rss_mb": 141.512, "state": "S (sleeping)", "threads": 106}, "restart_count": 914, "restart_count_per_hour": 24, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 915}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.608, "sample_count": 2647, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1944954, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:16:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.164, "last_frame_pushed_at": "2026-07-06T07:16:42+09:00", "last_frame_read_age_sec": 4.299, "last_frame_read_at": "2026-07-06T07:16:42+09:00", "mjpeg": {"age_sec": 4.147, "exists": true, "mtime": "2026-07-06T07:16:42.319909+09:00", "mtime_epoch": 1783289802.319909, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184171}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:16:46+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9553.637, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.989, "pushed": 5.989, "read": 5.955}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.532, "count": 179, "max_ms": 42.068, "p95_ms": 30.612}, "gstreamer_push": {"avg_ms": 1.077, "count": 179, "max_ms": 1.202, "p95_ms": 1.164}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.715, "count": 179, "max_ms": 71.101, "p95_ms": 63.12}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.969, "count": 179, "max_ms": 86.243, "p95_ms": 78.679}, "yolo11_640_fire_11class": {"avg_ms": 49.213, "count": 179, "max_ms": 67.696, "p95_ms": 62.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.124, "count": 179, "max_ms": 81.344, "p95_ms": 78.727}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.998, "count": 179, "max_ms": 176.363, "p95_ms": 155.719}, "read_interval": {"avg_ms": 144.475, "count": 178, "max_ms": 185.067, "p95_ms": 167.378}, "triton_wait": {"avg_ms": 60.996, "count": 179, "max_ms": 86.255, "p95_ms": 79.508}, "visualize": {"avg_ms": 4.464, "count": 179, "max_ms": 6.793, "p95_ms": 5.253}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16244}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16458.227, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56258742f700] left block unavailable for requested intra mode [h264 @ 0x56258742f700] error while decoding MB 0 17, bytestream 2090 [h264 @ 0x5625874579c0] cabac decode of qscale diff failed at 41 38 [h264 @ 0x5625874579c0] error while decoding MB 41 38, bytestream 3833 [h264 @ 0x5625873bb100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5625873bb100] error while decoding MB 0 10, bytestream 2269 [h264 @ 0x562588045480] left block unavailable for requested intra mode", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 822, "h264_decode_error_count": 1982, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:25:48+09:00", "last_error_summary": "[h264 @ 0x56258742f700] left block unavailable for requested intra mode [h264 @ 0x56258742f700] error while decoding MB 0 17, bytestream 2090 [h264 @ 0x5625874579c0] cabac decode of qscale diff failed at 41 38 [h264 @ 0x5625874579c0] error while decoding MB 41 38, bytestream 3833 [h264 @ 0x5625873bb100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5625873bb100] error while decoding MB 0 10, bytestream 2269 [h264 @ 0x562588045480] left block unavailable for requested intra mode", "last_exit_at": "2026-07-06T07:16:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 544.02, "last_started_at": "2026-07-06T07:16:44+09:00", "pid": 152604, "process": {"exists": false, "pid": 152604}, "restart_count": 914, "restart_count_per_hour": 20, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 915}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.92, "sample_count": 2231, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1949398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:25:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T07:25:48+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T07:25:48+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-06T07:25:48.425423+09:00", "mtime_epoch": 1783290348.425423, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203050}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:25:48+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9555.699, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 225, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.506, "pushed": 7.506, "read": 7.506}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.221, "count": 224, "max_ms": 33.148, "p95_ms": 29.672}, "gstreamer_push": {"avg_ms": 1.117, "count": 225, "max_ms": 1.327, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.958, "count": 224, "max_ms": 66.534, "p95_ms": 62.233}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.305, "count": 224, "max_ms": 83.329, "p95_ms": 79.065}, "yolo11_640_fire_11class": {"avg_ms": 50.783, "count": 224, "max_ms": 68.502, "p95_ms": 62.757}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.689, "count": 224, "max_ms": 83.489, "p95_ms": 77.286}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.558, "count": 225, "max_ms": 151.155, "p95_ms": 147.738}, "read_interval": {"avg_ms": 133.466, "count": 225, "max_ms": 164.546, "p95_ms": 160.917}, "triton_wait": {"avg_ms": 61.148, "count": 224, "max_ms": 83.502, "p95_ms": 79.387}, "visualize": {"avg_ms": 4.255, "count": 224, "max_ms": 6.509, "p95_ms": 5.0}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16244}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16457.758, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:25:48+09:00", "last_error_summary": "[h264 @ 0x56258742f700] left block unavailable for requested intra mode [h264 @ 0x56258742f700] error while decoding MB 0 17, bytestream 2090 [h264 @ 0x5625874579c0] cabac decode of qscale diff failed at 41 38 [h264 @ 0x5625874579c0] error while decoding MB 41 38, bytestream 3833 [h264 @ 0x5625873bb100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5625873bb100] error while decoding MB 0 10, bytestream 2269 [h264 @ 0x562588045480] left block unavailable for requested intra mode", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 822, "h264_decode_error_count": 1982, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:25:48+09:00", "last_error_summary": "[h264 @ 0x56258742f700] left block unavailable for requested intra mode [h264 @ 0x56258742f700] error while decoding MB 0 17, bytestream 2090 [h264 @ 0x5625874579c0] cabac decode of qscale diff failed at 41 38 [h264 @ 0x5625874579c0] error while decoding MB 41 38, bytestream 3833 [h264 @ 0x5625873bb100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5625873bb100] error while decoding MB 0 10, bytestream 2269 [h264 @ 0x562588045480] left block unavailable for requested intra mode", "last_exit_at": "2026-07-06T07:25:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 544.107, "last_started_at": "2026-07-06T07:16:44+09:00", "pid": 152604, "process": {"exists": false, "pid": 152604}, "restart_count": 914, "restart_count_per_hour": 20, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 915}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.937, "sample_count": 2231, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1949399, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:25:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T07:25:48+09:00", "last_frame_read_age_sec": 0.153, "last_frame_read_at": "2026-07-06T07:25:48+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-06T07:25:48.425423+09:00", "mtime_epoch": 1783290348.425423, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203050}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:25:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9555.699, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 225, "read_frames": 224, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.525, "pushed": 7.525, "read": 7.492}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.241, "count": 225, "max_ms": 33.148, "p95_ms": 29.795}, "gstreamer_push": {"avg_ms": 1.117, "count": 225, "max_ms": 1.327, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.944, "count": 225, "max_ms": 66.534, "p95_ms": 62.233}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.363, "count": 225, "max_ms": 83.329, "p95_ms": 79.065}, "yolo11_640_fire_11class": {"avg_ms": 50.78, "count": 225, "max_ms": 68.502, "p95_ms": 62.757}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.681, "count": 225, "max_ms": 83.489, "p95_ms": 77.286}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.676, "count": 225, "max_ms": 151.155, "p95_ms": 147.738}, "read_interval": {"avg_ms": 133.464, "count": 224, "max_ms": 164.546, "p95_ms": 160.917}, "triton_wait": {"avg_ms": 61.197, "count": 225, "max_ms": 83.502, "p95_ms": 79.387}, "visualize": {"avg_ms": 4.259, "count": 225, "max_ms": 6.509, "p95_ms": 5.011}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16244}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16457.828, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 822, "h264_decode_error_count": 1982, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:25:48+09:00", "last_error_summary": "[h264 @ 0x56258742f700] left block unavailable for requested intra mode [h264 @ 0x56258742f700] error while decoding MB 0 17, bytestream 2090 [h264 @ 0x5625874579c0] cabac decode of qscale diff failed at 41 38 [h264 @ 0x5625874579c0] error while decoding MB 41 38, bytestream 3833 [h264 @ 0x5625873bb100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5625873bb100] error while decoding MB 0 10, bytestream 2269 [h264 @ 0x562588045480] left block unavailable for requested intra mode", "last_exit_at": "2026-07-06T07:25:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 544.112, "last_started_at": "2026-07-06T07:16:44+09:00", "pid": 152604, "process": {"exists": false, "pid": 152604}, "restart_count": 914, "restart_count_per_hour": 20, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 915}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.937, "sample_count": 2231, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1949399, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:25:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T07:25:48+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-06T07:25:48+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-06T07:25:48.425423+09:00", "mtime_epoch": 1783290348.425423, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203050}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:25:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9555.699, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 225, "read_frames": 224, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.524, "pushed": 7.524, "read": 7.49}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.241, "count": 225, "max_ms": 33.148, "p95_ms": 29.795}, "gstreamer_push": {"avg_ms": 1.117, "count": 225, "max_ms": 1.327, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.944, "count": 225, "max_ms": 66.534, "p95_ms": 62.233}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.363, "count": 225, "max_ms": 83.329, "p95_ms": 79.065}, "yolo11_640_fire_11class": {"avg_ms": 50.78, "count": 225, "max_ms": 68.502, "p95_ms": 62.757}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.681, "count": 225, "max_ms": 83.489, "p95_ms": 77.286}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.676, "count": 225, "max_ms": 151.155, "p95_ms": 147.738}, "read_interval": {"avg_ms": 133.464, "count": 224, "max_ms": 164.546, "p95_ms": 160.917}, "triton_wait": {"avg_ms": 61.197, "count": 225, "max_ms": 83.502, "p95_ms": 79.387}, "visualize": {"avg_ms": 4.259, "count": 225, "max_ms": 6.509, "p95_ms": 5.011}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16244}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16563.477, "limit_mb": 8796093022207.996}, "detail": {"pid": 152831, "restart_count": 915, "start_count": 916}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 822, "h264_decode_error_count": 1982, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:25:48+09:00", "last_error_summary": "[h264 @ 0x56258742f700] left block unavailable for requested intra mode [h264 @ 0x56258742f700] error while decoding MB 0 17, bytestream 2090 [h264 @ 0x5625874579c0] cabac decode of qscale diff failed at 41 38 [h264 @ 0x5625874579c0] error while decoding MB 41 38, bytestream 3833 [h264 @ 0x5625873bb100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5625873bb100] error while decoding MB 0 10, bytestream 2269 [h264 @ 0x562588045480] left block unavailable for requested intra mode", "last_exit_at": "2026-07-06T07:25:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:25:50+09:00", "pid": 152831, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.414, "name": "ffmpeg", "pid": 152831, "rss_mb": 136.414, "state": "S (sleeping)", "threads": 106}, "restart_count": 915, "restart_count_per_hour": 21, "rtsp_timeout_count": 500, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 916}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.1, "sample_count": 2191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1949399, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:25:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.371, "last_frame_pushed_at": "2026-07-06T07:25:48+09:00", "last_frame_read_age_sec": 4.519, "last_frame_read_at": "2026-07-06T07:25:48+09:00", "mjpeg": {"age_sec": 4.353, "exists": true, "mtime": "2026-07-06T07:25:48.585427+09:00", "mtime_epoch": 1783290348.5854266, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204096}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:25:52+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9555.699, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 195, "pushed_frames": 195, "read_frames": 194, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.511, "pushed": 6.511, "read": 6.477}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.177, "count": 195, "max_ms": 33.148, "p95_ms": 29.838}, "gstreamer_push": {"avg_ms": 1.117, "count": 195, "max_ms": 1.327, "p95_ms": 1.257}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.748, "count": 195, "max_ms": 66.059, "p95_ms": 62.233}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.855, "count": 195, "max_ms": 83.329, "p95_ms": 78.631}, "yolo11_640_fire_11class": {"avg_ms": 50.406, "count": 195, "max_ms": 66.225, "p95_ms": 62.757}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.134, "count": 195, "max_ms": 80.119, "p95_ms": 77.004}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.004, "count": 195, "max_ms": 151.155, "p95_ms": 146.615}, "read_interval": {"avg_ms": 131.829, "count": 194, "max_ms": 164.546, "p95_ms": 159.214}, "triton_wait": {"avg_ms": 60.746, "count": 195, "max_ms": 83.341, "p95_ms": 79.078}, "visualize": {"avg_ms": 4.219, "count": 195, "max_ms": 6.509, "p95_ms": 5.197}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16255}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16633.531, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559c8cc93a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559c8cc93a40] error while decoding MB 0 7, bytestream 4211 [h264 @ 0x559c8cca7c00] left block unavailable for requested intra mode [h264 @ 0x559c8cca7c00] error while decoding MB 0 34, bytestream 36613 [h264 @ 0x559c8d778580] left block unavailable for requested intra mode [h264 @ 0x559c8d778580] error while decoding MB 0 29, bytestream 3065 [h264 @ 0x559c8d916280] error while decoding MB 39 22, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 823, "h264_decode_error_count": 1986, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:28:35+09:00", "last_error_summary": "[h264 @ 0x559c8cc93a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559c8cc93a40] error while decoding MB 0 7, bytestream 4211 [h264 @ 0x559c8cca7c00] left block unavailable for requested intra mode [h264 @ 0x559c8cca7c00] error while decoding MB 0 34, bytestream 36613 [h264 @ 0x559c8d778580] left block unavailable for requested intra mode [h264 @ 0x559c8d778580] error while decoding MB 0 29, bytestream 3065 [h264 @ 0x559c8d916280] error while decoding MB 39 22, bytestream -7", "last_exit_at": "2026-07-06T07:25:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.304, "last_started_at": "2026-07-06T07:25:50+09:00", "pid": 152831, "process": {"exists": false, "pid": 152831}, "restart_count": 915, "restart_count_per_hour": 20, "rtsp_timeout_count": 502, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 916}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.069, "sample_count": 2218, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1950654, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:28:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T07:28:35+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T07:28:35+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-06T07:28:35.189303+09:00", "mtime_epoch": 1783290515.1893034, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200657}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:28:35+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9556.215, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.305, "pushed": 3.305, "read": 3.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.286, "count": 99, "max_ms": 36.869, "p95_ms": 35.461}, "gstreamer_push": {"avg_ms": 1.174, "count": 99, "max_ms": 1.414, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.855, "count": 99, "max_ms": 66.668, "p95_ms": 63.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.324, "count": 99, "max_ms": 77.639, "p95_ms": 71.335}, "yolo11_640_fire_11class": {"avg_ms": 46.897, "count": 99, "max_ms": 66.135, "p95_ms": 62.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.272, "count": 99, "max_ms": 76.516, "p95_ms": 73.192}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.664, "count": 99, "max_ms": 109.243, "p95_ms": 107.134}, "read_interval": {"avg_ms": 302.85, "count": 99, "max_ms": 19666.84, "p95_ms": 123.152}, "triton_wait": {"avg_ms": 56.552, "count": 99, "max_ms": 77.651, "p95_ms": 74.848}, "visualize": {"avg_ms": 3.299, "count": 99, "max_ms": 4.387, "p95_ms": 4.1}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16255}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16633.293, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:28:35+09:00", "last_error_summary": "[h264 @ 0x559c8cc93a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559c8cc93a40] error while decoding MB 0 7, bytestream 4211 [h264 @ 0x559c8cca7c00] left block unavailable for requested intra mode [h264 @ 0x559c8cca7c00] error while decoding MB 0 34, bytestream 36613 [h264 @ 0x559c8d778580] left block unavailable for requested intra mode [h264 @ 0x559c8d778580] error while decoding MB 0 29, bytestream 3065 [h264 @ 0x559c8d916280] error while decoding MB 39 22, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 823, "h264_decode_error_count": 1986, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:28:35+09:00", "last_error_summary": "[h264 @ 0x559c8cc93a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559c8cc93a40] error while decoding MB 0 7, bytestream 4211 [h264 @ 0x559c8cca7c00] left block unavailable for requested intra mode [h264 @ 0x559c8cca7c00] error while decoding MB 0 34, bytestream 36613 [h264 @ 0x559c8d778580] left block unavailable for requested intra mode [h264 @ 0x559c8d778580] error while decoding MB 0 29, bytestream 3065 [h264 @ 0x559c8d916280] error while decoding MB 39 22, bytestream -7", "last_exit_at": "2026-07-06T07:28:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.353, "last_started_at": "2026-07-06T07:25:50+09:00", "pid": 152831, "process": {"exists": false, "pid": 152831}, "restart_count": 915, "restart_count_per_hour": 20, "rtsp_timeout_count": 502, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 916}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.069, "sample_count": 2219, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1950655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:28:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T07:28:35+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-06T07:28:35+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-06T07:28:35.189303+09:00", "mtime_epoch": 1783290515.1893034, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200657}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:28:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9556.215, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.3, "pushed": 3.333, "read": 3.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.292, "count": 99, "max_ms": 36.869, "p95_ms": 35.461}, "gstreamer_push": {"avg_ms": 1.175, "count": 100, "max_ms": 1.414, "p95_ms": 1.303}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.94, "count": 99, "max_ms": 66.668, "p95_ms": 63.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.435, "count": 99, "max_ms": 77.639, "p95_ms": 71.335}, "yolo11_640_fire_11class": {"avg_ms": 46.999, "count": 99, "max_ms": 66.135, "p95_ms": 62.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.366, "count": 99, "max_ms": 76.516, "p95_ms": 73.192}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.783, "count": 99, "max_ms": 109.243, "p95_ms": 107.134}, "read_interval": {"avg_ms": 302.85, "count": 99, "max_ms": 19666.84, "p95_ms": 123.152}, "triton_wait": {"avg_ms": 56.654, "count": 99, "max_ms": 77.651, "p95_ms": 74.848}, "visualize": {"avg_ms": 3.304, "count": 99, "max_ms": 4.387, "p95_ms": 4.1}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16255}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16633.277, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 823, "h264_decode_error_count": 1986, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:28:35+09:00", "last_error_summary": "[h264 @ 0x559c8cc93a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559c8cc93a40] error while decoding MB 0 7, bytestream 4211 [h264 @ 0x559c8cca7c00] left block unavailable for requested intra mode [h264 @ 0x559c8cca7c00] error while decoding MB 0 34, bytestream 36613 [h264 @ 0x559c8d778580] left block unavailable for requested intra mode [h264 @ 0x559c8d778580] error while decoding MB 0 29, bytestream 3065 [h264 @ 0x559c8d916280] error while decoding MB 39 22, bytestream -7", "last_exit_at": "2026-07-06T07:28:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.36, "last_started_at": "2026-07-06T07:25:50+09:00", "pid": 152831, "process": {"exists": false, "pid": 152831}, "restart_count": 915, "restart_count_per_hour": 20, "rtsp_timeout_count": 502, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 916}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.069, "sample_count": 2219, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1950655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:28:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T07:28:35+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T07:28:35+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T07:28:35.189303+09:00", "mtime_epoch": 1783290515.1893034, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200657}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:28:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9556.215, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.292, "count": 99, "max_ms": 36.869, "p95_ms": 35.461}, "gstreamer_push": {"avg_ms": 1.174, "count": 99, "max_ms": 1.414, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.94, "count": 99, "max_ms": 66.668, "p95_ms": 63.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.435, "count": 99, "max_ms": 77.639, "p95_ms": 71.335}, "yolo11_640_fire_11class": {"avg_ms": 46.999, "count": 99, "max_ms": 66.135, "p95_ms": 62.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.366, "count": 99, "max_ms": 76.516, "p95_ms": 73.192}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.783, "count": 99, "max_ms": 109.243, "p95_ms": 107.134}, "read_interval": {"avg_ms": 302.85, "count": 99, "max_ms": 19666.84, "p95_ms": 123.152}, "triton_wait": {"avg_ms": 56.654, "count": 99, "max_ms": 77.651, "p95_ms": 74.848}, "visualize": {"avg_ms": 3.304, "count": 99, "max_ms": 4.387, "p95_ms": 4.1}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16255}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16740.809, "limit_mb": 8796093022207.996}, "detail": {"pid": 152998, "restart_count": 916, "start_count": 917}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 823, "h264_decode_error_count": 1986, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:28:35+09:00", "last_error_summary": "[h264 @ 0x559c8cc93a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559c8cc93a40] error while decoding MB 0 7, bytestream 4211 [h264 @ 0x559c8cca7c00] left block unavailable for requested intra mode [h264 @ 0x559c8cca7c00] error while decoding MB 0 34, bytestream 36613 [h264 @ 0x559c8d778580] left block unavailable for requested intra mode [h264 @ 0x559c8d778580] error while decoding MB 0 29, bytestream 3065 [h264 @ 0x559c8d916280] error while decoding MB 39 22, bytestream -7", "last_exit_at": "2026-07-06T07:28:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:28:37+09:00", "pid": 152998, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.262, "name": "ffmpeg", "pid": 152998, "rss_mb": 139.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 916, "restart_count_per_hour": 21, "rtsp_timeout_count": 502, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 917}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.06, "sample_count": 2185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1950655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:28:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.578, "last_frame_pushed_at": "2026-07-06T07:28:35+09:00", "last_frame_read_age_sec": 4.682, "last_frame_read_at": "2026-07-06T07:28:35+09:00", "mjpeg": {"age_sec": 4.562, "exists": true, "mtime": "2026-07-06T07:28:35.301306+09:00", "mtime_epoch": 1783290515.301306, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199452}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:28:39+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9556.215, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.869, "pushed": 1.869, "read": 1.835}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.904, "count": 56, "max_ms": 36.869, "p95_ms": 36.323}, "gstreamer_push": {"avg_ms": 1.17, "count": 56, "max_ms": 1.414, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.397, "count": 56, "max_ms": 66.668, "p95_ms": 63.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.226, "count": 56, "max_ms": 77.639, "p95_ms": 74.836}, "yolo11_640_fire_11class": {"avg_ms": 47.85, "count": 56, "max_ms": 66.135, "p95_ms": 62.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.981, "count": 56, "max_ms": 73.192, "p95_ms": 72.237}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.249, "count": 56, "max_ms": 109.243, "p95_ms": 107.134}, "read_interval": {"avg_ms": 461.14, "count": 55, "max_ms": 19666.84, "p95_ms": 123.382}, "triton_wait": {"avg_ms": 57.511, "count": 56, "max_ms": 77.651, "p95_ms": 74.848}, "visualize": {"avg_ms": 3.307, "count": 56, "max_ms": 4.387, "p95_ms": 4.1}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 102, "tracking_summary": 20404}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16578.309, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556d95c08280] error while decoding MB 75 33, bytestream -7 [in#0/rtsp @ 0x556d950a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d950a2e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:34:11+09:00", "last_error_summary": "[h264 @ 0x556d95c08280] error while decoding MB 75 33, bytestream -7 [in#0/rtsp @ 0x556d950a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d950a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T06:53:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2459.058, "last_started_at": "2026-07-06T06:53:12+09:00", "pid": 151690, "process": {"exists": false, "pid": 151690}, "restart_count": 146, "restart_count_per_hour": 1, "rtsp_timeout_count": 250, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.13, "sample_count": 2869, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2448546, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:34:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T07:34:11+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T07:34:11+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-06T07:34:11.396955+09:00", "mtime_epoch": 1783290851.3969545, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218689}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T07:34:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2793.266, "name": "celery", "pid": 267, "rss_mb": 2717.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 118, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.935, "pushed": 3.935, "read": 3.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.3, "count": 118, "max_ms": 12.24, "p95_ms": 9.004}, "gstreamer_push": {"avg_ms": 1.331, "count": 118, "max_ms": 2.105, "p95_ms": 1.477}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.681, "count": 118, "max_ms": 66.565, "p95_ms": 62.484}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.637, "count": 118, "max_ms": 76.885, "p95_ms": 70.972}, "yolo11_640_fire_11class": {"avg_ms": 49.082, "count": 118, "max_ms": 64.278, "p95_ms": 61.599}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.318, "count": 118, "max_ms": 74.8, "p95_ms": 70.397}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.574, "count": 118, "max_ms": 109.821, "p95_ms": 89.73}, "read_interval": {"avg_ms": 254.184, "count": 118, "max_ms": 19315.834, "p95_ms": 108.153}, "triton_wait": {"avg_ms": 57.205, "count": 118, "max_ms": 76.899, "p95_ms": 72.609}, "visualize": {"avg_ms": 3.71, "count": 118, "max_ms": 4.863, "p95_ms": 4.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 102, "tracking_summary": 20404}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16577.703, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:34:11+09:00", "last_error_summary": "[h264 @ 0x556d95c08280] error while decoding MB 75 33, bytestream -7 [in#0/rtsp @ 0x556d950a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d950a2e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:34:11+09:00", "last_error_summary": "[h264 @ 0x556d95c08280] error while decoding MB 75 33, bytestream -7 [in#0/rtsp @ 0x556d950a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d950a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:34:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2459.101, "last_started_at": "2026-07-06T06:53:12+09:00", "pid": 151690, "process": {"exists": false, "pid": 151690}, "restart_count": 146, "restart_count_per_hour": 1, "rtsp_timeout_count": 250, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.13, "sample_count": 2870, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2448547, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:34:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T07:34:11+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-06T07:34:11+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T07:34:11.396955+09:00", "mtime_epoch": 1783290851.3969545, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218689}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T07:34:11+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2793.266, "name": "celery", "pid": 267, "rss_mb": 2717.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.941, "pushed": 3.941, "read": 3.907}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.299, "count": 118, "max_ms": 12.24, "p95_ms": 9.004}, "gstreamer_push": {"avg_ms": 1.329, "count": 118, "max_ms": 2.105, "p95_ms": 1.477}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.866, "count": 118, "max_ms": 68.285, "p95_ms": 62.901}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.624, "count": 118, "max_ms": 76.885, "p95_ms": 70.972}, "yolo11_640_fire_11class": {"avg_ms": 49.221, "count": 118, "max_ms": 64.278, "p95_ms": 61.623}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.549, "count": 118, "max_ms": 75.984, "p95_ms": 70.432}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.776, "count": 118, "max_ms": 109.821, "p95_ms": 91.593}, "read_interval": {"avg_ms": 255.624, "count": 117, "max_ms": 19315.834, "p95_ms": 108.153}, "triton_wait": {"avg_ms": 57.413, "count": 118, "max_ms": 76.899, "p95_ms": 72.708}, "visualize": {"avg_ms": 3.708, "count": 118, "max_ms": 4.863, "p95_ms": 4.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 102, "tracking_summary": 20404}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16577.52, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:34:11+09:00", "last_error_summary": "[h264 @ 0x556d95c08280] error while decoding MB 75 33, bytestream -7 [in#0/rtsp @ 0x556d950a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d950a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:34:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2459.109, "last_started_at": "2026-07-06T06:53:12+09:00", "pid": 151690, "process": {"exists": false, "pid": 151690}, "restart_count": 146, "restart_count_per_hour": 1, "rtsp_timeout_count": 250, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.13, "sample_count": 2870, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2448547, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:34:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T07:34:11+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T07:34:11+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-06T07:34:11.396955+09:00", "mtime_epoch": 1783290851.3969545, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218689}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T07:34:11+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2793.266, "name": "celery", "pid": 267, "rss_mb": 2717.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.94, "pushed": 3.94, "read": 3.906}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.299, "count": 118, "max_ms": 12.24, "p95_ms": 9.004}, "gstreamer_push": {"avg_ms": 1.329, "count": 118, "max_ms": 2.105, "p95_ms": 1.477}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.866, "count": 118, "max_ms": 68.285, "p95_ms": 62.901}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.624, "count": 118, "max_ms": 76.885, "p95_ms": 70.972}, "yolo11_640_fire_11class": {"avg_ms": 49.221, "count": 118, "max_ms": 64.278, "p95_ms": 61.623}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.549, "count": 118, "max_ms": 75.984, "p95_ms": 70.432}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.776, "count": 118, "max_ms": 109.821, "p95_ms": 91.593}, "read_interval": {"avg_ms": 255.624, "count": 117, "max_ms": 19315.834, "p95_ms": 108.153}, "triton_wait": {"avg_ms": 57.413, "count": 118, "max_ms": 76.899, "p95_ms": 72.708}, "visualize": {"avg_ms": 3.708, "count": 118, "max_ms": 4.863, "p95_ms": 4.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 102, "tracking_summary": 20404}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16757.352, "limit_mb": 8796093022207.996}, "detail": {"pid": 153105, "restart_count": 147, "start_count": 148}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:34:11+09:00", "last_error_summary": "[h264 @ 0x556d95c08280] error while decoding MB 75 33, bytestream -7 [in#0/rtsp @ 0x556d950a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d950a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:34:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:34:14+09:00", "pid": 153105, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.996, "name": "ffmpeg", "pid": 153105, "rss_mb": 209.996, "state": "S (sleeping)", "threads": 106}, "restart_count": 147, "restart_count_per_hour": 2, "rtsp_timeout_count": 250, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.133, "sample_count": 2821, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2448547, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:34:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.519, "last_frame_pushed_at": "2026-07-06T07:34:11+09:00", "last_frame_read_age_sec": 4.614, "last_frame_read_at": "2026-07-06T07:34:11+09:00", "mjpeg": {"age_sec": 4.504, "exists": true, "mtime": "2026-07-06T07:34:11.504957+09:00", "mtime_epoch": 1783290851.504957, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 215701}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T07:34:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2793.266, "name": "celery", "pid": 267, "rss_mb": 2717.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 68, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.304, "pushed": 2.304, "read": 2.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.344, "count": 69, "max_ms": 12.24, "p95_ms": 9.345}, "gstreamer_push": {"avg_ms": 1.325, "count": 69, "max_ms": 2.105, "p95_ms": 1.477}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.749, "count": 69, "max_ms": 68.285, "p95_ms": 63.006}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.894, "count": 69, "max_ms": 72.596, "p95_ms": 66.125}, "yolo11_640_fire_11class": {"avg_ms": 48.989, "count": 69, "max_ms": 62.479, "p95_ms": 61.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.275, "count": 69, "max_ms": 75.984, "p95_ms": 71.637}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.9, "count": 69, "max_ms": 109.821, "p95_ms": 89.73}, "read_interval": {"avg_ms": 373.996, "count": 68, "max_ms": 19315.834, "p95_ms": 105.939}, "triton_wait": {"avg_ms": 57.295, "count": 69, "max_ms": 75.996, "p95_ms": 72.609}, "visualize": {"avg_ms": 3.691, "count": 69, "max_ms": 4.863, "p95_ms": 4.016}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16312}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16453.172, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56521417ec80] error while decoding MB 49 24, bytestream -24 [h264 @ 0x5652140f31c0] error while decoding MB 64 2, bytestream -9 [h264 @ 0x565214c431c0] error while decoding MB 23 6, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 824, "h264_decode_error_count": 1989, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:40:50+09:00", "last_error_summary": "[h264 @ 0x56521417ec80] error while decoding MB 49 24, bytestream -24 [h264 @ 0x5652140f31c0] error while decoding MB 64 2, bytestream -9 [h264 @ 0x565214c431c0] error while decoding MB 23 6, bytestream -7", "last_exit_at": "2026-07-06T07:28:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 732.431, "last_started_at": "2026-07-06T07:28:37+09:00", "pid": 152998, "process": {"exists": false, "pid": 152998}, "restart_count": 916, "restart_count_per_hour": 16, "rtsp_timeout_count": 502, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 917}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.839, "sample_count": 2798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1957469, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:40:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T07:40:50+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-06T07:40:50+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-06T07:40:50.241104+09:00", "mtime_epoch": 1783291250.2411036, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189003}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:40:50+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9559.566, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 280, "pushed_frames": 280, "read_frames": 280, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.345, "pushed": 9.345, "read": 9.345}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.491, "count": 280, "max_ms": 33.84, "p95_ms": 27.651}, "gstreamer_push": {"avg_ms": 1.191, "count": 280, "max_ms": 1.916, "p95_ms": 1.262}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.374, "count": 280, "max_ms": 70.345, "p95_ms": 63.131}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.101, "count": 280, "max_ms": 84.061, "p95_ms": 75.823}, "yolo11_640_fire_11class": {"avg_ms": 50.201, "count": 280, "max_ms": 68.735, "p95_ms": 62.303}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.477, "count": 280, "max_ms": 75.846, "p95_ms": 71.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.069, "count": 280, "max_ms": 117.48, "p95_ms": 109.217}, "read_interval": {"avg_ms": 107.179, "count": 280, "max_ms": 136.17, "p95_ms": 123.919}, "triton_wait": {"avg_ms": 58.73, "count": 280, "max_ms": 84.073, "p95_ms": 75.859}, "visualize": {"avg_ms": 3.319, "count": 280, "max_ms": 4.865, "p95_ms": 3.614}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16312}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16452.59, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:40:50+09:00", "last_error_summary": "[h264 @ 0x56521417ec80] error while decoding MB 49 24, bytestream -24 [h264 @ 0x5652140f31c0] error while decoding MB 64 2, bytestream -9 [h264 @ 0x565214c431c0] error while decoding MB 23 6, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 824, "h264_decode_error_count": 1989, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:40:50+09:00", "last_error_summary": "[h264 @ 0x56521417ec80] error while decoding MB 49 24, bytestream -24 [h264 @ 0x5652140f31c0] error while decoding MB 64 2, bytestream -9 [h264 @ 0x565214c431c0] error while decoding MB 23 6, bytestream -7", "last_exit_at": "2026-07-06T07:40:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 732.486, "last_started_at": "2026-07-06T07:28:37+09:00", "pid": 152998, "process": {"exists": false, "pid": 152998}, "restart_count": 916, "restart_count_per_hour": 16, "rtsp_timeout_count": 502, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 917}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.839, "sample_count": 2798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1957470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:40:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T07:40:50+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-06T07:40:50+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T07:40:50.241104+09:00", "mtime_epoch": 1783291250.2411036, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189003}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:40:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9559.566, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 280, "pushed_frames": 280, "read_frames": 279, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.365, "pushed": 9.365, "read": 9.331}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.494, "count": 280, "max_ms": 33.84, "p95_ms": 27.651}, "gstreamer_push": {"avg_ms": 1.191, "count": 280, "max_ms": 1.916, "p95_ms": 1.262}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.386, "count": 280, "max_ms": 70.345, "p95_ms": 63.277}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.081, "count": 280, "max_ms": 84.061, "p95_ms": 75.823}, "yolo11_640_fire_11class": {"avg_ms": 50.23, "count": 280, "max_ms": 68.735, "p95_ms": 62.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.47, "count": 280, "max_ms": 75.846, "p95_ms": 71.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.064, "count": 280, "max_ms": 117.48, "p95_ms": 109.217}, "read_interval": {"avg_ms": 107.133, "count": 279, "max_ms": 136.17, "p95_ms": 124.467}, "triton_wait": {"avg_ms": 58.722, "count": 280, "max_ms": 84.073, "p95_ms": 75.859}, "visualize": {"avg_ms": 3.32, "count": 280, "max_ms": 4.865, "p95_ms": 3.614}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16312}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16452.621, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 824, "h264_decode_error_count": 1989, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:40:50+09:00", "last_error_summary": "[h264 @ 0x56521417ec80] error while decoding MB 49 24, bytestream -24 [h264 @ 0x5652140f31c0] error while decoding MB 64 2, bytestream -9 [h264 @ 0x565214c431c0] error while decoding MB 23 6, bytestream -7", "last_exit_at": "2026-07-06T07:40:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 732.494, "last_started_at": "2026-07-06T07:28:37+09:00", "pid": 152998, "process": {"exists": false, "pid": 152998}, "restart_count": 916, "restart_count_per_hour": 16, "rtsp_timeout_count": 502, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 917}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.839, "sample_count": 2798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1957470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:40:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T07:40:50+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-06T07:40:50+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T07:40:50.241104+09:00", "mtime_epoch": 1783291250.2411036, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189003}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:40:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9559.566, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 280, "pushed_frames": 280, "read_frames": 279, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.362, "pushed": 9.362, "read": 9.329}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.494, "count": 280, "max_ms": 33.84, "p95_ms": 27.651}, "gstreamer_push": {"avg_ms": 1.191, "count": 280, "max_ms": 1.916, "p95_ms": 1.262}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.386, "count": 280, "max_ms": 70.345, "p95_ms": 63.277}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.081, "count": 280, "max_ms": 84.061, "p95_ms": 75.823}, "yolo11_640_fire_11class": {"avg_ms": 50.23, "count": 280, "max_ms": 68.735, "p95_ms": 62.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.47, "count": 280, "max_ms": 75.846, "p95_ms": 71.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.064, "count": 280, "max_ms": 117.48, "p95_ms": 109.217}, "read_interval": {"avg_ms": 107.133, "count": 279, "max_ms": 136.17, "p95_ms": 124.467}, "triton_wait": {"avg_ms": 58.722, "count": 280, "max_ms": 84.073, "p95_ms": 75.859}, "visualize": {"avg_ms": 3.32, "count": 280, "max_ms": 4.865, "p95_ms": 3.614}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16312}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16557.727, "limit_mb": 8796093022207.996}, "detail": {"pid": 153257, "restart_count": 917, "start_count": 918}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 824, "h264_decode_error_count": 1989, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:40:50+09:00", "last_error_summary": "[h264 @ 0x56521417ec80] error while decoding MB 49 24, bytestream -24 [h264 @ 0x5652140f31c0] error while decoding MB 64 2, bytestream -9 [h264 @ 0x565214c431c0] error while decoding MB 23 6, bytestream -7", "last_exit_at": "2026-07-06T07:40:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:40:52+09:00", "pid": 153257, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.277, "name": "ffmpeg", "pid": 153257, "rss_mb": 136.277, "state": "S (sleeping)", "threads": 106}, "restart_count": 917, "restart_count_per_hour": 17, "rtsp_timeout_count": 502, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 918}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.708, "sample_count": 2757, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1957470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:40:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.458, "last_frame_pushed_at": "2026-07-06T07:40:50+09:00", "last_frame_read_age_sec": 4.564, "last_frame_read_at": "2026-07-06T07:40:50+09:00", "mjpeg": {"age_sec": 4.443, "exists": true, "mtime": "2026-07-06T07:40:50.357103+09:00", "mtime_epoch": 1783291250.3571029, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181642}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:40:54+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9559.566, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 239, "pushed_frames": 239, "read_frames": 238, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.967, "pushed": 7.967, "read": 7.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.468, "count": 238, "max_ms": 33.84, "p95_ms": 27.797}, "gstreamer_push": {"avg_ms": 1.191, "count": 239, "max_ms": 1.916, "p95_ms": 1.266}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.541, "count": 238, "max_ms": 70.345, "p95_ms": 63.391}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.187, "count": 238, "max_ms": 84.061, "p95_ms": 76.066}, "yolo11_640_fire_11class": {"avg_ms": 50.435, "count": 238, "max_ms": 68.735, "p95_ms": 62.607}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.596, "count": 238, "max_ms": 75.846, "p95_ms": 71.937}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.152, "count": 239, "max_ms": 117.48, "p95_ms": 109.365}, "read_interval": {"avg_ms": 107.232, "count": 238, "max_ms": 136.17, "p95_ms": 124.467}, "triton_wait": {"avg_ms": 58.845, "count": 238, "max_ms": 84.073, "p95_ms": 76.079}, "visualize": {"avg_ms": 3.324, "count": 238, "max_ms": 4.865, "p95_ms": 3.629}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16345}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16458.168, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 10, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a476cc0d00] error while decoding MB 18 34, bytestream -7 [h264 @ 0x55a477945b40] cabac decode of qscale diff failed at 9 11 [h264 @ 0x55a477945b40] error while decoding MB 9 11, bytestream 4235 [h264 @ 0x55a477876cc0] cabac decode of qscale diff failed at 12 31 [h264 @ 0x55a477876cc0] error while decoding MB 12 31, bytestream 3710 [h264 @ 0x55a477945b40] error while decoding MB 27 22, bytestream -7 [h264 @ 0x55a4777a7e40] left block unavailable for requested intra4x4 mode -1 [h264 @ ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 825, "h264_decode_error_count": 1999, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:48:17+09:00", "last_error_summary": "[h264 @ 0x55a476cc0d00] error while decoding MB 18 34, bytestream -7 [h264 @ 0x55a477945b40] cabac decode of qscale diff failed at 9 11 [h264 @ 0x55a477945b40] error while decoding MB 9 11, bytestream 4235 [h264 @ 0x55a477876cc0] cabac decode of qscale diff failed at 12 31 [h264 @ 0x55a477876cc0] error while decoding MB 12 31, bytestream 3710 [h264 @ 0x55a477945b40] error while decoding MB 27 22, bytestream -7 [h264 @ 0x55a4777a7e40] left block unavailable for requested intra4x4 mode -1 [h264 @ ", "last_exit_at": "2026-07-06T07:40:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 444.78, "last_started_at": "2026-07-06T07:40:52+09:00", "pid": 153257, "process": {"exists": false, "pid": 153257}, "restart_count": 917, "restart_count_per_hour": 13, "rtsp_timeout_count": 504, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 918}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.138, "sample_count": 2613, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1961403, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:48:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-06T07:48:17+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-06T07:48:17+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-06T07:48:17.525461+09:00", "mtime_epoch": 1783291697.525461, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196452}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:48:17+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9561.371, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 103, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.438, "pushed": 3.438, "read": 3.438}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.93, "count": 103, "max_ms": 36.808, "p95_ms": 35.111}, "gstreamer_push": {"avg_ms": 1.167, "count": 103, "max_ms": 1.66, "p95_ms": 1.264}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.492, "count": 103, "max_ms": 71.326, "p95_ms": 61.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.25, "count": 103, "max_ms": 88.769, "p95_ms": 68.68}, "yolo11_640_fire_11class": {"avg_ms": 46.734, "count": 103, "max_ms": 70.753, "p95_ms": 61.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.198, "count": 103, "max_ms": 80.29, "p95_ms": 71.529}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.196, "count": 103, "max_ms": 121.212, "p95_ms": 106.64}, "read_interval": {"avg_ms": 291.084, "count": 103, "max_ms": 10124.555, "p95_ms": 124.225}, "triton_wait": {"avg_ms": 55.495, "count": 103, "max_ms": 88.783, "p95_ms": 73.903}, "visualize": {"avg_ms": 3.282, "count": 103, "max_ms": 4.257, "p95_ms": 3.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16345}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16458.012, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:48:17+09:00", "last_error_summary": "[h264 @ 0x55a476cc0d00] error while decoding MB 18 34, bytestream -7 [h264 @ 0x55a477945b40] cabac decode of qscale diff failed at 9 11 [h264 @ 0x55a477945b40] error while decoding MB 9 11, bytestream 4235 [h264 @ 0x55a477876cc0] cabac decode of qscale diff failed at 12 31 [h264 @ 0x55a477876cc0] error while decoding MB 12 31, bytestream 3710 [h264 @ 0x55a477945b40] error while decoding MB 27 22, bytestream -7 [h264 @ 0x55a4777a7e40] left block unavailable for requested intra4x4 mode -1 [h264 @ ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 825, "h264_decode_error_count": 1999, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:48:17+09:00", "last_error_summary": "[h264 @ 0x55a476cc0d00] error while decoding MB 18 34, bytestream -7 [h264 @ 0x55a477945b40] cabac decode of qscale diff failed at 9 11 [h264 @ 0x55a477945b40] error while decoding MB 9 11, bytestream 4235 [h264 @ 0x55a477876cc0] cabac decode of qscale diff failed at 12 31 [h264 @ 0x55a477876cc0] error while decoding MB 12 31, bytestream 3710 [h264 @ 0x55a477945b40] error while decoding MB 27 22, bytestream -7 [h264 @ 0x55a4777a7e40] left block unavailable for requested intra4x4 mode -1 [h264 @ ", "last_exit_at": "2026-07-06T07:48:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 444.833, "last_started_at": "2026-07-06T07:40:52+09:00", "pid": 153257, "process": {"exists": false, "pid": 153257}, "restart_count": 917, "restart_count_per_hour": 13, "rtsp_timeout_count": 504, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 918}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.138, "sample_count": 2614, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1961404, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:48:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T07:48:17+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T07:48:17+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T07:48:17.525461+09:00", "mtime_epoch": 1783291697.525461, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196452}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:48:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9561.371, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 103, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.433, "pushed": 3.433, "read": 3.433}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.991, "count": 103, "max_ms": 36.808, "p95_ms": 35.111}, "gstreamer_push": {"avg_ms": 1.169, "count": 103, "max_ms": 1.66, "p95_ms": 1.265}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.555, "count": 103, "max_ms": 71.326, "p95_ms": 61.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.4, "count": 103, "max_ms": 88.769, "p95_ms": 68.68}, "yolo11_640_fire_11class": {"avg_ms": 46.8, "count": 103, "max_ms": 70.753, "p95_ms": 61.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.32, "count": 103, "max_ms": 80.29, "p95_ms": 71.529}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.423, "count": 103, "max_ms": 121.212, "p95_ms": 106.64}, "read_interval": {"avg_ms": 291.084, "count": 103, "max_ms": 10124.555, "p95_ms": 124.225}, "triton_wait": {"avg_ms": 55.644, "count": 103, "max_ms": 88.783, "p95_ms": 73.903}, "visualize": {"avg_ms": 3.294, "count": 103, "max_ms": 4.725, "p95_ms": 3.892}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16345}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16458.004, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 825, "h264_decode_error_count": 1999, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:48:17+09:00", "last_error_summary": "[h264 @ 0x55a476cc0d00] error while decoding MB 18 34, bytestream -7 [h264 @ 0x55a477945b40] cabac decode of qscale diff failed at 9 11 [h264 @ 0x55a477945b40] error while decoding MB 9 11, bytestream 4235 [h264 @ 0x55a477876cc0] cabac decode of qscale diff failed at 12 31 [h264 @ 0x55a477876cc0] error while decoding MB 12 31, bytestream 3710 [h264 @ 0x55a477945b40] error while decoding MB 27 22, bytestream -7 [h264 @ 0x55a4777a7e40] left block unavailable for requested intra4x4 mode -1 [h264 @ ", "last_exit_at": "2026-07-06T07:48:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 444.842, "last_started_at": "2026-07-06T07:40:52+09:00", "pid": 153257, "process": {"exists": false, "pid": 153257}, "restart_count": 917, "restart_count_per_hour": 13, "rtsp_timeout_count": 504, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 918}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.138, "sample_count": 2614, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1961404, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:48:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T07:48:17+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-06T07:48:17+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T07:48:17.525461+09:00", "mtime_epoch": 1783291697.525461, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196452}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:48:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9561.371, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.442, "pushed": 3.442, "read": 3.408}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.991, "count": 103, "max_ms": 36.808, "p95_ms": 35.111}, "gstreamer_push": {"avg_ms": 1.169, "count": 103, "max_ms": 1.66, "p95_ms": 1.265}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.555, "count": 103, "max_ms": 71.326, "p95_ms": 61.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.4, "count": 103, "max_ms": 88.769, "p95_ms": 68.68}, "yolo11_640_fire_11class": {"avg_ms": 46.8, "count": 103, "max_ms": 70.753, "p95_ms": 61.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.32, "count": 103, "max_ms": 80.29, "p95_ms": 71.529}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.423, "count": 103, "max_ms": 121.212, "p95_ms": 106.64}, "read_interval": {"avg_ms": 293.009, "count": 102, "max_ms": 10124.555, "p95_ms": 124.225}, "triton_wait": {"avg_ms": 55.644, "count": 103, "max_ms": 88.783, "p95_ms": 73.903}, "visualize": {"avg_ms": 3.294, "count": 103, "max_ms": 4.725, "p95_ms": 3.892}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16345}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16563.637, "limit_mb": 8796093022207.996}, "detail": {"pid": 153364, "restart_count": 918, "start_count": 919}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 825, "h264_decode_error_count": 1999, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:48:17+09:00", "last_error_summary": "[h264 @ 0x55a476cc0d00] error while decoding MB 18 34, bytestream -7 [h264 @ 0x55a477945b40] cabac decode of qscale diff failed at 9 11 [h264 @ 0x55a477945b40] error while decoding MB 9 11, bytestream 4235 [h264 @ 0x55a477876cc0] cabac decode of qscale diff failed at 12 31 [h264 @ 0x55a477876cc0] error while decoding MB 12 31, bytestream 3710 [h264 @ 0x55a477945b40] error while decoding MB 27 22, bytestream -7 [h264 @ 0x55a4777a7e40] left block unavailable for requested intra4x4 mode -1 [h264 @ ", "last_exit_at": "2026-07-06T07:48:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:48:19+09:00", "pid": 153364, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.363, "name": "ffmpeg", "pid": 153364, "rss_mb": 136.363, "state": "S (sleeping)", "threads": 106}, "restart_count": 918, "restart_count_per_hour": 14, "rtsp_timeout_count": 504, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 919}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.138, "sample_count": 2575, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1961404, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:48:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.168, "last_frame_pushed_at": "2026-07-06T07:48:17+09:00", "last_frame_read_age_sec": 4.274, "last_frame_read_at": "2026-07-06T07:48:17+09:00", "mjpeg": {"age_sec": 4.152, "exists": true, "mtime": "2026-07-06T07:48:17.641462+09:00", "mtime_epoch": 1783291697.6414616, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197058}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:48:21+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9561.371, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.1, "pushed": 2.133, "read": 2.1}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.607, "count": 63, "max_ms": 36.808, "p95_ms": 32.655}, "gstreamer_push": {"avg_ms": 1.179, "count": 64, "max_ms": 1.66, "p95_ms": 1.315}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.917, "count": 63, "max_ms": 64.312, "p95_ms": 61.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.36, "count": 63, "max_ms": 78.131, "p95_ms": 68.68}, "yolo11_640_fire_11class": {"avg_ms": 48.218, "count": 63, "max_ms": 63.789, "p95_ms": 62.223}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.999, "count": 63, "max_ms": 73.966, "p95_ms": 71.855}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.126, "count": 63, "max_ms": 111.359, "p95_ms": 108.834}, "read_interval": {"avg_ms": 410.287, "count": 63, "max_ms": 10124.555, "p95_ms": 127.97}, "triton_wait": {"avg_ms": 56.676, "count": 63, "max_ms": 78.143, "p95_ms": 73.979}, "visualize": {"avg_ms": 3.322, "count": 63, "max_ms": 4.725, "p95_ms": 3.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16462.789, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5612b894b5c0] reference picture missing during reorder [h264 @ 0x5612b894b5c0] Missing reference picture, default is 65541 [h264 @ 0x5612b8952d00] mmco: unref short failure [h264 @ 0x5612b8952d00] illegal short term buffer state detected [h264 @ 0x5612b89433c0] error while decoding MB 63 43, bytestream -8 [h264 @ 0x5612b925c100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612b925c100] error while decoding MB 0 19, bytestream 12066 [h264 @ 0x5612b8807d80] error whil", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 826, "h264_decode_error_count": 2004, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:54:21+09:00", "last_error_summary": "[h264 @ 0x5612b894b5c0] reference picture missing during reorder [h264 @ 0x5612b894b5c0] Missing reference picture, default is 65541 [h264 @ 0x5612b8952d00] mmco: unref short failure [h264 @ 0x5612b8952d00] illegal short term buffer state detected [h264 @ 0x5612b89433c0] error while decoding MB 63 43, bytestream -8 [h264 @ 0x5612b925c100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612b925c100] error while decoding MB 0 19, bytestream 12066 [h264 @ 0x5612b8807d80] error whil", "last_exit_at": "2026-07-06T07:48:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 362.186, "last_started_at": "2026-07-06T07:48:19+09:00", "pid": 153364, "process": {"exists": false, "pid": 153364}, "restart_count": 918, "restart_count_per_hour": 11, "rtsp_timeout_count": 506, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 919}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.776, "sample_count": 2617, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1964596, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:54:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.092, "last_frame_pushed_at": "2026-07-06T07:54:21+09:00", "last_frame_read_age_sec": 0.081, "last_frame_read_at": "2026-07-06T07:54:21+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-06T07:54:21.900494+09:00", "mtime_epoch": 1783292061.9004943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185694}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:54:21+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9562.918, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.409, "pushed": 3.409, "read": 3.409}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.163, "count": 102, "max_ms": 40.231, "p95_ms": 28.379}, "gstreamer_push": {"avg_ms": 1.206, "count": 102, "max_ms": 1.402, "p95_ms": 1.299}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.556, "count": 102, "max_ms": 65.718, "p95_ms": 64.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.167, "count": 102, "max_ms": 77.421, "p95_ms": 71.848}, "yolo11_640_fire_11class": {"avg_ms": 50.275, "count": 102, "max_ms": 65.541, "p95_ms": 62.983}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.56, "count": 102, "max_ms": 75.942, "p95_ms": 71.608}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.407, "count": 102, "max_ms": 119.009, "p95_ms": 107.061}, "read_interval": {"avg_ms": 293.538, "count": 102, "max_ms": 10126.15, "p95_ms": 124.425}, "triton_wait": {"avg_ms": 58.293, "count": 102, "max_ms": 77.434, "p95_ms": 73.161}, "visualize": {"avg_ms": 3.325, "count": 102, "max_ms": 4.408, "p95_ms": 3.526}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16462.707, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:54:21+09:00", "last_error_summary": "[h264 @ 0x5612b894b5c0] reference picture missing during reorder [h264 @ 0x5612b894b5c0] Missing reference picture, default is 65541 [h264 @ 0x5612b8952d00] mmco: unref short failure [h264 @ 0x5612b8952d00] illegal short term buffer state detected [h264 @ 0x5612b89433c0] error while decoding MB 63 43, bytestream -8 [h264 @ 0x5612b925c100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612b925c100] error while decoding MB 0 19, bytestream 12066 [h264 @ 0x5612b8807d80] error whil", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 826, "h264_decode_error_count": 2004, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:54:21+09:00", "last_error_summary": "[h264 @ 0x5612b894b5c0] reference picture missing during reorder [h264 @ 0x5612b894b5c0] Missing reference picture, default is 65541 [h264 @ 0x5612b8952d00] mmco: unref short failure [h264 @ 0x5612b8952d00] illegal short term buffer state detected [h264 @ 0x5612b89433c0] error while decoding MB 63 43, bytestream -8 [h264 @ 0x5612b925c100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612b925c100] error while decoding MB 0 19, bytestream 12066 [h264 @ 0x5612b8807d80] error whil", "last_exit_at": "2026-07-06T07:54:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 362.215, "last_started_at": "2026-07-06T07:48:19+09:00", "pid": 153364, "process": {"exists": false, "pid": 153364}, "restart_count": 918, "restart_count_per_hour": 11, "rtsp_timeout_count": 506, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 919}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.776, "sample_count": 2618, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1964597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:54:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T07:54:21+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T07:54:21+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T07:54:21.900494+09:00", "mtime_epoch": 1783292061.9004943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185694}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:54:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9562.918, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.439, "pushed": 3.439, "read": 3.406}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.201, "count": 103, "max_ms": 40.231, "p95_ms": 28.404}, "gstreamer_push": {"avg_ms": 1.208, "count": 103, "max_ms": 1.402, "p95_ms": 1.3}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.542, "count": 103, "max_ms": 65.718, "p95_ms": 64.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.169, "count": 103, "max_ms": 77.421, "p95_ms": 71.848}, "yolo11_640_fire_11class": {"avg_ms": 50.263, "count": 103, "max_ms": 65.541, "p95_ms": 62.983}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.523, "count": 103, "max_ms": 75.942, "p95_ms": 71.608}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.478, "count": 103, "max_ms": 119.009, "p95_ms": 107.061}, "read_interval": {"avg_ms": 293.538, "count": 102, "max_ms": 10126.15, "p95_ms": 124.425}, "triton_wait": {"avg_ms": 58.274, "count": 103, "max_ms": 77.434, "p95_ms": 73.161}, "visualize": {"avg_ms": 3.364, "count": 103, "max_ms": 7.366, "p95_ms": 3.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16462.641, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 826, "h264_decode_error_count": 2004, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:54:21+09:00", "last_error_summary": "[h264 @ 0x5612b894b5c0] reference picture missing during reorder [h264 @ 0x5612b894b5c0] Missing reference picture, default is 65541 [h264 @ 0x5612b8952d00] mmco: unref short failure [h264 @ 0x5612b8952d00] illegal short term buffer state detected [h264 @ 0x5612b89433c0] error while decoding MB 63 43, bytestream -8 [h264 @ 0x5612b925c100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612b925c100] error while decoding MB 0 19, bytestream 12066 [h264 @ 0x5612b8807d80] error whil", "last_exit_at": "2026-07-06T07:54:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 362.224, "last_started_at": "2026-07-06T07:48:19+09:00", "pid": 153364, "process": {"exists": false, "pid": 153364}, "restart_count": 918, "restart_count_per_hour": 11, "rtsp_timeout_count": 506, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 919}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.776, "sample_count": 2618, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1964597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:54:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T07:54:21+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T07:54:21+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T07:54:21.900494+09:00", "mtime_epoch": 1783292061.9004943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185694}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:54:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9562.918, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.438, "pushed": 3.438, "read": 3.405}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.201, "count": 103, "max_ms": 40.231, "p95_ms": 28.404}, "gstreamer_push": {"avg_ms": 1.208, "count": 103, "max_ms": 1.402, "p95_ms": 1.3}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.542, "count": 103, "max_ms": 65.718, "p95_ms": 64.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.169, "count": 103, "max_ms": 77.421, "p95_ms": 71.848}, "yolo11_640_fire_11class": {"avg_ms": 50.263, "count": 103, "max_ms": 65.541, "p95_ms": 62.983}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.523, "count": 103, "max_ms": 75.942, "p95_ms": 71.608}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.478, "count": 103, "max_ms": 119.009, "p95_ms": 107.061}, "read_interval": {"avg_ms": 293.538, "count": 102, "max_ms": 10126.15, "p95_ms": 124.425}, "triton_wait": {"avg_ms": 58.274, "count": 103, "max_ms": 77.434, "p95_ms": 73.161}, "visualize": {"avg_ms": 3.364, "count": 103, "max_ms": 7.366, "p95_ms": 3.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16568.301, "limit_mb": 8796093022207.996}, "detail": {"pid": 153471, "restart_count": 919, "start_count": 920}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 826, "h264_decode_error_count": 2004, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:54:21+09:00", "last_error_summary": "[h264 @ 0x5612b894b5c0] reference picture missing during reorder [h264 @ 0x5612b894b5c0] Missing reference picture, default is 65541 [h264 @ 0x5612b8952d00] mmco: unref short failure [h264 @ 0x5612b8952d00] illegal short term buffer state detected [h264 @ 0x5612b89433c0] error while decoding MB 63 43, bytestream -8 [h264 @ 0x5612b925c100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612b925c100] error while decoding MB 0 19, bytestream 12066 [h264 @ 0x5612b8807d80] error whil", "last_exit_at": "2026-07-06T07:54:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:54:24+09:00", "pid": 153471, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.73, "name": "ffmpeg", "pid": 153471, "rss_mb": 136.73, "state": "S (sleeping)", "threads": 106}, "restart_count": 919, "restart_count_per_hour": 12, "rtsp_timeout_count": 506, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 920}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.776, "sample_count": 2575, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1964597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:54:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.57, "last_frame_pushed_at": "2026-07-06T07:54:21+09:00", "last_frame_read_age_sec": 4.672, "last_frame_read_at": "2026-07-06T07:54:21+09:00", "mjpeg": {"age_sec": 4.553, "exists": true, "mtime": "2026-07-06T07:54:22.016495+09:00", "mtime_epoch": 1783292062.0164955, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185984}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:54:26+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9562.918, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.003, "pushed": 2.003, "read": 1.97}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.274, "count": 60, "max_ms": 40.231, "p95_ms": 28.404}, "gstreamer_push": {"avg_ms": 1.216, "count": 60, "max_ms": 1.402, "p95_ms": 1.3}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.821, "count": 60, "max_ms": 65.553, "p95_ms": 62.138}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.83, "count": 60, "max_ms": 77.421, "p95_ms": 71.848}, "yolo11_640_fire_11class": {"avg_ms": 50.526, "count": 60, "max_ms": 65.541, "p95_ms": 62.418}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.175, "count": 60, "max_ms": 75.942, "p95_ms": 72.687}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.36, "count": 60, "max_ms": 111.837, "p95_ms": 107.061}, "read_interval": {"avg_ms": 430.204, "count": 59, "max_ms": 10126.15, "p95_ms": 126.652}, "triton_wait": {"avg_ms": 59.008, "count": 60, "max_ms": 77.434, "p95_ms": 73.951}, "visualize": {"avg_ms": 3.391, "count": 60, "max_ms": 7.366, "p95_ms": 3.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16388}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16679.617, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557e633220c0] cabac decode of qscale diff failed at 50 31 [h264 @ 0x557e633220c0] error while decoding MB 50 31, bytestream 4639 [h264 @ 0x557e63ce30c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557e63ce30c0] error while decoding MB 0 13, bytestream 3609 [h264 @ 0x557e63c14300] cabac decode of qscale diff failed at 31 33 [h264 @ 0x557e63c14300] error while decoding MB 31 33, bytestream 35321 [in#0/rtsp @ 0x557e63248e00] Error during demuxing: Connection timed out ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 827, "h264_decode_error_count": 2009, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:58:48+09:00", "last_error_summary": "[h264 @ 0x557e633220c0] cabac decode of qscale diff failed at 50 31 [h264 @ 0x557e633220c0] error while decoding MB 50 31, bytestream 4639 [h264 @ 0x557e63ce30c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557e63ce30c0] error while decoding MB 0 13, bytestream 3609 [h264 @ 0x557e63c14300] cabac decode of qscale diff failed at 31 33 [h264 @ 0x557e63c14300] error while decoding MB 31 33, bytestream 35321 [in#0/rtsp @ 0x557e63248e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-06T07:54:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 263.901, "last_started_at": "2026-07-06T07:54:24+09:00", "pid": 153471, "process": {"exists": false, "pid": 153471}, "restart_count": 919, "restart_count_per_hour": 10, "rtsp_timeout_count": 508, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 920}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.51, "sample_count": 2193, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1966654, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:58:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T07:58:48+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-06T07:58:48+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T07:58:48.419469+09:00", "mtime_epoch": 1783292328.419469, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205441}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:58:48+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9563.949, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 81, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.713, "pushed": 2.713, "read": 2.713}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.3, "count": 81, "max_ms": 32.22, "p95_ms": 31.094}, "gstreamer_push": {"avg_ms": 1.163, "count": 81, "max_ms": 1.477, "p95_ms": 1.26}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.061, "count": 81, "max_ms": 67.545, "p95_ms": 63.427}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.797, "count": 81, "max_ms": 84.986, "p95_ms": 78.731}, "yolo11_640_fire_11class": {"avg_ms": 50.4, "count": 81, "max_ms": 68.069, "p95_ms": 62.65}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.395, "count": 81, "max_ms": 82.627, "p95_ms": 73.84}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.403, "count": 81, "max_ms": 172.847, "p95_ms": 142.152}, "read_interval": {"avg_ms": 369.459, "count": 81, "max_ms": 20571.543, "p95_ms": 160.59}, "triton_wait": {"avg_ms": 59.572, "count": 81, "max_ms": 84.998, "p95_ms": 78.743}, "visualize": {"avg_ms": 3.331, "count": 81, "max_ms": 6.195, "p95_ms": 3.877}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.858, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16388}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16679.27, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:58:48+09:00", "last_error_summary": "[h264 @ 0x557e633220c0] cabac decode of qscale diff failed at 50 31 [h264 @ 0x557e633220c0] error while decoding MB 50 31, bytestream 4639 [h264 @ 0x557e63ce30c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557e63ce30c0] error while decoding MB 0 13, bytestream 3609 [h264 @ 0x557e63c14300] cabac decode of qscale diff failed at 31 33 [h264 @ 0x557e63c14300] error while decoding MB 31 33, bytestream 35321 [in#0/rtsp @ 0x557e63248e00] Error during demuxing: Connection timed out ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 827, "h264_decode_error_count": 2009, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:58:48+09:00", "last_error_summary": "[h264 @ 0x557e633220c0] cabac decode of qscale diff failed at 50 31 [h264 @ 0x557e633220c0] error while decoding MB 50 31, bytestream 4639 [h264 @ 0x557e63ce30c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557e63ce30c0] error while decoding MB 0 13, bytestream 3609 [h264 @ 0x557e63c14300] cabac decode of qscale diff failed at 31 33 [h264 @ 0x557e63c14300] error while decoding MB 31 33, bytestream 35321 [in#0/rtsp @ 0x557e63248e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-06T07:58:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 263.95, "last_started_at": "2026-07-06T07:54:24+09:00", "pid": 153471, "process": {"exists": false, "pid": 153471}, "restart_count": 919, "restart_count_per_hour": 10, "rtsp_timeout_count": 508, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 920}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.51, "sample_count": 2194, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1966655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:58:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T07:58:48+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T07:58:48+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T07:58:48.419469+09:00", "mtime_epoch": 1783292328.419469, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205441}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:58:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9563.949, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 82, "pushed_frames": 82, "read_frames": 81, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.744, "pushed": 2.744, "read": 2.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.365, "count": 82, "max_ms": 32.22, "p95_ms": 31.458}, "gstreamer_push": {"avg_ms": 1.165, "count": 82, "max_ms": 1.477, "p95_ms": 1.279}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.073, "count": 82, "max_ms": 67.545, "p95_ms": 63.427}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.807, "count": 82, "max_ms": 84.986, "p95_ms": 78.731}, "yolo11_640_fire_11class": {"avg_ms": 50.427, "count": 82, "max_ms": 68.069, "p95_ms": 62.65}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.376, "count": 82, "max_ms": 82.627, "p95_ms": 73.84}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.357, "count": 82, "max_ms": 172.847, "p95_ms": 142.152}, "read_interval": {"avg_ms": 369.459, "count": 81, "max_ms": 20571.543, "p95_ms": 160.59}, "triton_wait": {"avg_ms": 59.56, "count": 82, "max_ms": 84.998, "p95_ms": 78.743}, "visualize": {"avg_ms": 3.342, "count": 82, "max_ms": 6.195, "p95_ms": 3.897}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16388}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16679.367, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 827, "h264_decode_error_count": 2009, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:58:48+09:00", "last_error_summary": "[h264 @ 0x557e633220c0] cabac decode of qscale diff failed at 50 31 [h264 @ 0x557e633220c0] error while decoding MB 50 31, bytestream 4639 [h264 @ 0x557e63ce30c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557e63ce30c0] error while decoding MB 0 13, bytestream 3609 [h264 @ 0x557e63c14300] cabac decode of qscale diff failed at 31 33 [h264 @ 0x557e63c14300] error while decoding MB 31 33, bytestream 35321 [in#0/rtsp @ 0x557e63248e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-06T07:58:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 263.958, "last_started_at": "2026-07-06T07:54:24+09:00", "pid": 153471, "process": {"exists": false, "pid": 153471}, "restart_count": 919, "restart_count_per_hour": 10, "rtsp_timeout_count": 508, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 920}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.51, "sample_count": 2194, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1966655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:58:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T07:58:48+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T07:58:48+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T07:58:48.419469+09:00", "mtime_epoch": 1783292328.419469, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205441}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:58:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9563.949, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 82, "pushed_frames": 82, "read_frames": 81, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.743, "pushed": 2.743, "read": 2.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.365, "count": 82, "max_ms": 32.22, "p95_ms": 31.458}, "gstreamer_push": {"avg_ms": 1.165, "count": 82, "max_ms": 1.477, "p95_ms": 1.279}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.073, "count": 82, "max_ms": 67.545, "p95_ms": 63.427}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.807, "count": 82, "max_ms": 84.986, "p95_ms": 78.731}, "yolo11_640_fire_11class": {"avg_ms": 50.427, "count": 82, "max_ms": 68.069, "p95_ms": 62.65}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.376, "count": 82, "max_ms": 82.627, "p95_ms": 73.84}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.357, "count": 82, "max_ms": 172.847, "p95_ms": 142.152}, "read_interval": {"avg_ms": 369.459, "count": 81, "max_ms": 20571.543, "p95_ms": 160.59}, "triton_wait": {"avg_ms": 59.56, "count": 82, "max_ms": 84.998, "p95_ms": 78.743}, "visualize": {"avg_ms": 3.342, "count": 82, "max_ms": 6.195, "p95_ms": 3.897}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16388}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16786.953, "limit_mb": 8796093022207.996}, "detail": {"pid": 153638, "restart_count": 920, "start_count": 921}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 827, "h264_decode_error_count": 2009, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:58:48+09:00", "last_error_summary": "[h264 @ 0x557e633220c0] cabac decode of qscale diff failed at 50 31 [h264 @ 0x557e633220c0] error while decoding MB 50 31, bytestream 4639 [h264 @ 0x557e63ce30c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557e63ce30c0] error while decoding MB 0 13, bytestream 3609 [h264 @ 0x557e63c14300] cabac decode of qscale diff failed at 31 33 [h264 @ 0x557e63c14300] error while decoding MB 31 33, bytestream 35321 [in#0/rtsp @ 0x557e63248e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-06T07:58:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:58:50+09:00", "pid": 153638, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.918, "name": "ffmpeg", "pid": 153638, "rss_mb": 138.918, "state": "S (sleeping)", "threads": 106}, "restart_count": 920, "restart_count_per_hour": 11, "rtsp_timeout_count": 508, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 921}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.597, "sample_count": 2153, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1966655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:58:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.36, "last_frame_pushed_at": "2026-07-06T07:58:48+09:00", "last_frame_read_age_sec": 4.461, "last_frame_read_at": "2026-07-06T07:58:48+09:00", "mjpeg": {"age_sec": 4.342, "exists": true, "mtime": "2026-07-06T07:58:48.531471+09:00", "mtime_epoch": 1783292328.5314705, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206196}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T07:58:52+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9563.949, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.638, "pushed": 1.638, "read": 1.604}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.375, "count": 49, "max_ms": 31.606, "p95_ms": 27.839}, "gstreamer_push": {"avg_ms": 1.18, "count": 49, "max_ms": 1.365, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.729, "count": 49, "max_ms": 63.805, "p95_ms": 63.427}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.199, "count": 49, "max_ms": 80.905, "p95_ms": 79.675}, "yolo11_640_fire_11class": {"avg_ms": 50.53, "count": 49, "max_ms": 65.126, "p95_ms": 62.779}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.317, "count": 49, "max_ms": 74.305, "p95_ms": 72.295}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.593, "count": 49, "max_ms": 113.748, "p95_ms": 113.226}, "read_interval": {"avg_ms": 532.487, "count": 48, "max_ms": 20571.543, "p95_ms": 126.506}, "triton_wait": {"avg_ms": 59.412, "count": 49, "max_ms": 80.917, "p95_ms": 79.687}, "visualize": {"avg_ms": 3.297, "count": 49, "max_ms": 4.234, "p95_ms": 3.635}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16619.785, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560c0c5cdb40] error while decoding MB 112 48, bytestream -6 [in#0/rtsp @ 0x560c0c39ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c0c39ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:59:42+09:00", "last_error_summary": "[h264 @ 0x560c0c5cdb40] error while decoding MB 112 48, bytestream -6 [in#0/rtsp @ 0x560c0c39ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c0c39ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:34:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 1528.26, "last_started_at": "2026-07-06T07:34:14+09:00", "pid": 153105, "process": {"exists": false, "pid": 153105}, "restart_count": 147, "restart_count_per_hour": 1, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.616, "sample_count": 2502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2463310, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:59:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T07:59:42+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T07:59:42+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T07:59:42.212118+09:00", "mtime_epoch": 1783292382.2121181, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221925}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T07:59:42+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2756.797, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 38, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.27, "pushed": 1.27, "read": 1.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.799, "count": 39, "max_ms": 10.329, "p95_ms": 9.418}, "gstreamer_push": {"avg_ms": 1.298, "count": 38, "max_ms": 1.44, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.847, "count": 39, "max_ms": 66.172, "p95_ms": 63.743}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.435, "count": 39, "max_ms": 84.958, "p95_ms": 84.153}, "yolo11_640_fire_11class": {"avg_ms": 50.46, "count": 39, "max_ms": 65.299, "p95_ms": 62.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.073, "count": 39, "max_ms": 79.31, "p95_ms": 74.434}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.845, "count": 38, "max_ms": 102.207, "p95_ms": 101.446}, "read_interval": {"avg_ms": 788.354, "count": 38, "max_ms": 16048.747, "p95_ms": 10558.842}, "triton_wait": {"avg_ms": 59.047, "count": 39, "max_ms": 84.97, "p95_ms": 84.165}, "visualize": {"avg_ms": 3.688, "count": 39, "max_ms": 4.172, "p95_ms": 3.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16619.621, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T07:59:42+09:00", "last_error_summary": "[h264 @ 0x560c0c5cdb40] error while decoding MB 112 48, bytestream -6 [in#0/rtsp @ 0x560c0c39ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c0c39ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:59:42+09:00", "last_error_summary": "[h264 @ 0x560c0c5cdb40] error while decoding MB 112 48, bytestream -6 [in#0/rtsp @ 0x560c0c39ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c0c39ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:59:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 1528.279, "last_started_at": "2026-07-06T07:34:14+09:00", "pid": 153105, "process": {"exists": false, "pid": 153105}, "restart_count": 147, "restart_count_per_hour": 1, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.616, "sample_count": 2503, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2463310, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:59:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T07:59:42+09:00", "last_frame_read_age_sec": 0.081, "last_frame_read_at": "2026-07-06T07:59:42+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-06T07:59:42.212118+09:00", "mtime_epoch": 1783292382.2121181, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221925}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T07:59:42+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2756.797, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 38, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.302, "pushed": 1.302, "read": 1.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.799, "count": 39, "max_ms": 10.329, "p95_ms": 9.418}, "gstreamer_push": {"avg_ms": 1.301, "count": 39, "max_ms": 1.44, "p95_ms": 1.438}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.847, "count": 39, "max_ms": 66.172, "p95_ms": 63.743}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.435, "count": 39, "max_ms": 84.958, "p95_ms": 84.153}, "yolo11_640_fire_11class": {"avg_ms": 50.46, "count": 39, "max_ms": 65.299, "p95_ms": 62.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.073, "count": 39, "max_ms": 79.31, "p95_ms": 74.434}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.739, "count": 39, "max_ms": 102.207, "p95_ms": 101.446}, "read_interval": {"avg_ms": 788.354, "count": 38, "max_ms": 16048.747, "p95_ms": 10558.842}, "triton_wait": {"avg_ms": 59.047, "count": 39, "max_ms": 84.97, "p95_ms": 84.165}, "visualize": {"avg_ms": 3.688, "count": 39, "max_ms": 4.172, "p95_ms": 3.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16619.566, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:59:42+09:00", "last_error_summary": "[h264 @ 0x560c0c5cdb40] error while decoding MB 112 48, bytestream -6 [in#0/rtsp @ 0x560c0c39ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c0c39ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:59:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 1528.287, "last_started_at": "2026-07-06T07:34:14+09:00", "pid": 153105, "process": {"exists": false, "pid": 153105}, "restart_count": 147, "restart_count_per_hour": 1, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.616, "sample_count": 2503, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2463310, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:59:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T07:59:42+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-06T07:59:42+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-06T07:59:42.212118+09:00", "mtime_epoch": 1783292382.2121181, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221925}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T07:59:42+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2756.797, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 38, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.302, "pushed": 1.302, "read": 1.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.799, "count": 39, "max_ms": 10.329, "p95_ms": 9.418}, "gstreamer_push": {"avg_ms": 1.301, "count": 39, "max_ms": 1.44, "p95_ms": 1.438}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.847, "count": 39, "max_ms": 66.172, "p95_ms": 63.743}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.435, "count": 39, "max_ms": 84.958, "p95_ms": 84.153}, "yolo11_640_fire_11class": {"avg_ms": 50.46, "count": 39, "max_ms": 65.299, "p95_ms": 62.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.073, "count": 39, "max_ms": 79.31, "p95_ms": 74.434}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.739, "count": 39, "max_ms": 102.207, "p95_ms": 101.446}, "read_interval": {"avg_ms": 788.354, "count": 38, "max_ms": 16048.747, "p95_ms": 10558.842}, "triton_wait": {"avg_ms": 59.047, "count": 39, "max_ms": 84.97, "p95_ms": 84.165}, "visualize": {"avg_ms": 3.688, "count": 39, "max_ms": 4.172, "p95_ms": 3.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16799.496, "limit_mb": 8796093022207.996}, "detail": {"pid": 153745, "restart_count": 148, "start_count": 149}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T07:59:42+09:00", "last_error_summary": "[h264 @ 0x560c0c5cdb40] error while decoding MB 112 48, bytestream -6 [in#0/rtsp @ 0x560c0c39ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c0c39ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:59:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T07:59:44+09:00", "pid": 153745, "process": {"exists": true, "fd_count": 5, "hwm_mb": 210.047, "name": "ffmpeg", "pid": 153745, "rss_mb": 210.047, "state": "S (sleeping)", "threads": 106}, "restart_count": 148, "restart_count_per_hour": 2, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.648, "sample_count": 2457, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2463310, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T07:59:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.564, "last_frame_pushed_at": "2026-07-06T07:59:42+09:00", "last_frame_read_age_sec": 4.639, "last_frame_read_at": "2026-07-06T07:59:42+09:00", "mjpeg": {"age_sec": 4.549, "exists": true, "mtime": "2026-07-06T07:59:42.296119+09:00", "mtime_epoch": 1783292382.2961192, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221818}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T07:59:46+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2756.797, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.599, "pushed": 0.599, "read": 0.599}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.866, "count": 13, "max_ms": 10.329, "p95_ms": 10.329}, "gstreamer_push": {"avg_ms": 1.325, "count": 13, "max_ms": 1.438, "p95_ms": 1.438}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.977, "count": 13, "max_ms": 66.172, "p95_ms": 66.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.626, "count": 13, "max_ms": 84.153, "p95_ms": 84.153}, "yolo11_640_fire_11class": {"avg_ms": 50.095, "count": 13, "max_ms": 65.299, "p95_ms": 65.299}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.082, "count": 13, "max_ms": 79.31, "p95_ms": 79.31}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.791, "count": 13, "max_ms": 101.446, "p95_ms": 101.446}, "read_interval": {"avg_ms": 2124.501, "count": 13, "max_ms": 16048.747, "p95_ms": 16048.747}, "triton_wait": {"avg_ms": 58.688, "count": 13, "max_ms": 84.165, "p95_ms": 84.165}, "visualize": {"avg_ms": 3.678, "count": 13, "max_ms": 4.172, "p95_ms": 4.172}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.699, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16415}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16605.566, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561f2a8d6c00] error while decoding MB 24 3, bytestream -6 [h264 @ 0x561f2a78ff00] error while decoding MB 34 40, bytestream -12 [in#0/rtsp @ 0x561f2a770e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f2a770e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 828, "h264_decode_error_count": 2011, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:04:48+09:00", "last_error_summary": "[h264 @ 0x561f2a8d6c00] error while decoding MB 24 3, bytestream -6 [h264 @ 0x561f2a78ff00] error while decoding MB 34 40, bytestream -12 [in#0/rtsp @ 0x561f2a770e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f2a770e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T07:58:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 357.574, "last_started_at": "2026-07-06T07:58:50+09:00", "pid": 153638, "process": {"exists": false, "pid": 153638}, "restart_count": 920, "restart_count_per_hour": 9, "rtsp_timeout_count": 510, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 921}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.083, "sample_count": 2660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1969861, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:04:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.089, "last_frame_pushed_at": "2026-07-06T08:04:48+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-06T08:04:48+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-06T08:04:48.372018+09:00", "mtime_epoch": 1783292688.372018, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:04:48+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9565.496, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 103, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.4, "pushed": 3.4, "read": 3.434}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.147, "count": 102, "max_ms": 29.93, "p95_ms": 27.995}, "gstreamer_push": {"avg_ms": 1.195, "count": 102, "max_ms": 1.349, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.391, "count": 102, "max_ms": 65.435, "p95_ms": 61.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.217, "count": 102, "max_ms": 80.62, "p95_ms": 75.687}, "yolo11_640_fire_11class": {"avg_ms": 49.703, "count": 102, "max_ms": 64.188, "p95_ms": 61.378}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.356, "count": 102, "max_ms": 74.238, "p95_ms": 72.353}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.243, "count": 102, "max_ms": 113.245, "p95_ms": 109.574}, "read_interval": {"avg_ms": 291.661, "count": 103, "max_ms": 10158.498, "p95_ms": 126.279}, "triton_wait": {"avg_ms": 58.155, "count": 102, "max_ms": 80.632, "p95_ms": 75.699}, "visualize": {"avg_ms": 3.316, "count": 102, "max_ms": 4.155, "p95_ms": 3.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16415}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16605.414, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:04:48+09:00", "last_error_summary": "[h264 @ 0x561f2a8d6c00] error while decoding MB 24 3, bytestream -6 [h264 @ 0x561f2a78ff00] error while decoding MB 34 40, bytestream -12 [in#0/rtsp @ 0x561f2a770e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f2a770e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 828, "h264_decode_error_count": 2011, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:04:48+09:00", "last_error_summary": "[h264 @ 0x561f2a8d6c00] error while decoding MB 24 3, bytestream -6 [h264 @ 0x561f2a78ff00] error while decoding MB 34 40, bytestream -12 [in#0/rtsp @ 0x561f2a770e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f2a770e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:04:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 357.6, "last_started_at": "2026-07-06T07:58:50+09:00", "pid": 153638, "process": {"exists": false, "pid": 153638}, "restart_count": 920, "restart_count_per_hour": 9, "rtsp_timeout_count": 510, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 921}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.083, "sample_count": 2661, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1969862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:04:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T08:04:48+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-06T08:04:48+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T08:04:48.372018+09:00", "mtime_epoch": 1783292688.372018, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:04:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9565.496, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.441, "pushed": 3.441, "read": 3.408}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.194, "count": 103, "max_ms": 31.018, "p95_ms": 28.051}, "gstreamer_push": {"avg_ms": 1.197, "count": 103, "max_ms": 1.393, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.362, "count": 103, "max_ms": 65.435, "p95_ms": 61.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.178, "count": 103, "max_ms": 80.62, "p95_ms": 75.687}, "yolo11_640_fire_11class": {"avg_ms": 49.67, "count": 103, "max_ms": 64.188, "p95_ms": 61.378}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.303, "count": 103, "max_ms": 74.238, "p95_ms": 72.353}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.259, "count": 103, "max_ms": 113.245, "p95_ms": 109.574}, "read_interval": {"avg_ms": 293.337, "count": 102, "max_ms": 10158.498, "p95_ms": 126.279}, "triton_wait": {"avg_ms": 58.097, "count": 103, "max_ms": 80.632, "p95_ms": 75.699}, "visualize": {"avg_ms": 3.335, "count": 103, "max_ms": 5.276, "p95_ms": 3.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16415}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16605.531, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 828, "h264_decode_error_count": 2011, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:04:48+09:00", "last_error_summary": "[h264 @ 0x561f2a8d6c00] error while decoding MB 24 3, bytestream -6 [h264 @ 0x561f2a78ff00] error while decoding MB 34 40, bytestream -12 [in#0/rtsp @ 0x561f2a770e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f2a770e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:04:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 357.608, "last_started_at": "2026-07-06T07:58:50+09:00", "pid": 153638, "process": {"exists": false, "pid": 153638}, "restart_count": 920, "restart_count_per_hour": 9, "rtsp_timeout_count": 510, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 921}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.083, "sample_count": 2661, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1969862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:04:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T08:04:48+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T08:04:48+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T08:04:48.372018+09:00", "mtime_epoch": 1783292688.372018, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:04:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9565.496, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.441, "pushed": 3.441, "read": 3.407}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.194, "count": 103, "max_ms": 31.018, "p95_ms": 28.051}, "gstreamer_push": {"avg_ms": 1.197, "count": 103, "max_ms": 1.393, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.362, "count": 103, "max_ms": 65.435, "p95_ms": 61.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.178, "count": 103, "max_ms": 80.62, "p95_ms": 75.687}, "yolo11_640_fire_11class": {"avg_ms": 49.67, "count": 103, "max_ms": 64.188, "p95_ms": 61.378}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.303, "count": 103, "max_ms": 74.238, "p95_ms": 72.353}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.259, "count": 103, "max_ms": 113.245, "p95_ms": 109.574}, "read_interval": {"avg_ms": 293.337, "count": 102, "max_ms": 10158.498, "p95_ms": 126.279}, "triton_wait": {"avg_ms": 58.097, "count": 103, "max_ms": 80.632, "p95_ms": 75.699}, "visualize": {"avg_ms": 3.335, "count": 103, "max_ms": 5.276, "p95_ms": 3.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16415}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16713.945, "limit_mb": 8796093022207.996}, "detail": {"pid": 153897, "restart_count": 921, "start_count": 922}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 828, "h264_decode_error_count": 2011, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:04:48+09:00", "last_error_summary": "[h264 @ 0x561f2a8d6c00] error while decoding MB 24 3, bytestream -6 [h264 @ 0x561f2a78ff00] error while decoding MB 34 40, bytestream -12 [in#0/rtsp @ 0x561f2a770e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f2a770e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:04:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:04:51+09:00", "pid": 153897, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.859, "name": "ffmpeg", "pid": 153897, "rss_mb": 138.859, "state": "S (sleeping)", "threads": 106}, "restart_count": 921, "restart_count_per_hour": 10, "rtsp_timeout_count": 510, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 922}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.131, "sample_count": 2614, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1969862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:04:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.945, "last_frame_pushed_at": "2026-07-06T08:04:48+09:00", "last_frame_read_age_sec": 5.041, "last_frame_read_at": "2026-07-06T08:04:48+09:00", "mjpeg": {"age_sec": 4.93, "exists": true, "mtime": "2026-07-06T08:04:48.480020+09:00", "mtime_epoch": 1783292688.4800196, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 212197}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:04:53+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9565.496, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.905, "pushed": 1.905, "read": 1.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.351, "count": 57, "max_ms": 31.018, "p95_ms": 28.136}, "gstreamer_push": {"avg_ms": 1.217, "count": 57, "max_ms": 1.393, "p95_ms": 1.341}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.179, "count": 57, "max_ms": 64.695, "p95_ms": 61.188}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.105, "count": 57, "max_ms": 80.62, "p95_ms": 75.687}, "yolo11_640_fire_11class": {"avg_ms": 49.722, "count": 57, "max_ms": 64.188, "p95_ms": 60.644}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.975, "count": 57, "max_ms": 74.045, "p95_ms": 70.475}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.039, "count": 57, "max_ms": 113.245, "p95_ms": 110.814}, "read_interval": {"avg_ms": 446.07, "count": 56, "max_ms": 10158.498, "p95_ms": 126.69}, "triton_wait": {"avg_ms": 57.662, "count": 57, "max_ms": 80.632, "p95_ms": 75.699}, "visualize": {"avg_ms": 3.348, "count": 57, "max_ms": 5.276, "p95_ms": 3.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16598.055, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55689c82e300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55689c82e300] error while decoding MB 0 6, bytestream 98260 [h264 @ 0x55689d322040] error while decoding MB 41 35, bytestream -5 [in#0/rtsp @ 0x55689c7a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55689c7a5e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 829, "h264_decode_error_count": 2013, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:09:08+09:00", "last_error_summary": "[h264 @ 0x55689c82e300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55689c82e300] error while decoding MB 0 6, bytestream 98260 [h264 @ 0x55689d322040] error while decoding MB 41 35, bytestream -5 [in#0/rtsp @ 0x55689c7a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55689c7a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:04:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 257.532, "last_started_at": "2026-07-06T08:04:51+09:00", "pid": 153897, "process": {"exists": false, "pid": 153897}, "restart_count": 921, "restart_count_per_hour": 8, "rtsp_timeout_count": 512, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 922}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.879, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1972080, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:09:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T08:09:08+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T08:09:08+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-06T08:09:08.895524+09:00", "mtime_epoch": 1783292948.8955243, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201878}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:09:08+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9566.527, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 103, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.442, "pushed": 3.442, "read": 3.442}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.941, "count": 103, "max_ms": 37.578, "p95_ms": 29.295}, "gstreamer_push": {"avg_ms": 1.181, "count": 103, "max_ms": 1.753, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.782, "count": 103, "max_ms": 68.692, "p95_ms": 62.856}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.138, "count": 103, "max_ms": 81.113, "p95_ms": 73.182}, "yolo11_640_fire_11class": {"avg_ms": 48.928, "count": 103, "max_ms": 64.427, "p95_ms": 61.957}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.197, "count": 103, "max_ms": 76.976, "p95_ms": 74.694}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.564, "count": 103, "max_ms": 114.343, "p95_ms": 109.652}, "read_interval": {"avg_ms": 290.827, "count": 103, "max_ms": 10131.645, "p95_ms": 126.462}, "triton_wait": {"avg_ms": 57.842, "count": 103, "max_ms": 81.125, "p95_ms": 75.858}, "visualize": {"avg_ms": 3.309, "count": 103, "max_ms": 4.745, "p95_ms": 3.573}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16597.512, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:09:08+09:00", "last_error_summary": "[h264 @ 0x55689c82e300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55689c82e300] error while decoding MB 0 6, bytestream 98260 [h264 @ 0x55689d322040] error while decoding MB 41 35, bytestream -5 [in#0/rtsp @ 0x55689c7a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55689c7a5e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 829, "h264_decode_error_count": 2013, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:09:08+09:00", "last_error_summary": "[h264 @ 0x55689c82e300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55689c82e300] error while decoding MB 0 6, bytestream 98260 [h264 @ 0x55689d322040] error while decoding MB 41 35, bytestream -5 [in#0/rtsp @ 0x55689c7a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55689c7a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:09:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 257.594, "last_started_at": "2026-07-06T08:04:51+09:00", "pid": 153897, "process": {"exists": false, "pid": 153897}, "restart_count": 921, "restart_count_per_hour": 8, "rtsp_timeout_count": 512, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 922}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.879, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1972081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:09:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T08:09:08+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T08:09:08+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T08:09:08.895524+09:00", "mtime_epoch": 1783292948.8955243, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201878}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:09:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9566.527, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.469, "pushed": 3.469, "read": 3.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.931, "count": 104, "max_ms": 37.578, "p95_ms": 29.295}, "gstreamer_push": {"avg_ms": 1.182, "count": 104, "max_ms": 1.753, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.98, "count": 104, "max_ms": 70.39, "p95_ms": 63.088}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.163, "count": 104, "max_ms": 81.113, "p95_ms": 73.182}, "yolo11_640_fire_11class": {"avg_ms": 49.028, "count": 104, "max_ms": 64.427, "p95_ms": 61.957}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.407, "count": 104, "max_ms": 76.976, "p95_ms": 75.026}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.722, "count": 104, "max_ms": 114.343, "p95_ms": 109.652}, "read_interval": {"avg_ms": 290.827, "count": 103, "max_ms": 10131.645, "p95_ms": 126.462}, "triton_wait": {"avg_ms": 58.007, "count": 104, "max_ms": 81.125, "p95_ms": 75.858}, "visualize": {"avg_ms": 3.312, "count": 104, "max_ms": 4.745, "p95_ms": 3.573}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16597.551, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 829, "h264_decode_error_count": 2013, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:09:08+09:00", "last_error_summary": "[h264 @ 0x55689c82e300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55689c82e300] error while decoding MB 0 6, bytestream 98260 [h264 @ 0x55689d322040] error while decoding MB 41 35, bytestream -5 [in#0/rtsp @ 0x55689c7a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55689c7a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:09:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 257.6, "last_started_at": "2026-07-06T08:04:51+09:00", "pid": 153897, "process": {"exists": false, "pid": 153897}, "restart_count": 921, "restart_count_per_hour": 8, "rtsp_timeout_count": 512, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 922}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.879, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1972081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:09:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T08:09:08+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-06T08:09:08+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T08:09:08.895524+09:00", "mtime_epoch": 1783292948.8955243, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201878}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:09:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9566.527, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.468, "pushed": 3.468, "read": 3.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.931, "count": 104, "max_ms": 37.578, "p95_ms": 29.295}, "gstreamer_push": {"avg_ms": 1.182, "count": 104, "max_ms": 1.753, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.98, "count": 104, "max_ms": 70.39, "p95_ms": 63.088}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.163, "count": 104, "max_ms": 81.113, "p95_ms": 73.182}, "yolo11_640_fire_11class": {"avg_ms": 49.028, "count": 104, "max_ms": 64.427, "p95_ms": 61.957}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.407, "count": 104, "max_ms": 76.976, "p95_ms": 75.026}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.722, "count": 104, "max_ms": 114.343, "p95_ms": 109.652}, "read_interval": {"avg_ms": 290.827, "count": 103, "max_ms": 10131.645, "p95_ms": 126.462}, "triton_wait": {"avg_ms": 58.007, "count": 104, "max_ms": 81.125, "p95_ms": 75.858}, "visualize": {"avg_ms": 3.312, "count": 104, "max_ms": 4.745, "p95_ms": 3.573}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16705.785, "limit_mb": 8796093022207.996}, "detail": {"pid": 154004, "restart_count": 922, "start_count": 923}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 829, "h264_decode_error_count": 2013, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:09:08+09:00", "last_error_summary": "[h264 @ 0x55689c82e300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55689c82e300] error while decoding MB 0 6, bytestream 98260 [h264 @ 0x55689d322040] error while decoding MB 41 35, bytestream -5 [in#0/rtsp @ 0x55689c7a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55689c7a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:09:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:09:11+09:00", "pid": 154004, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.902, "name": "ffmpeg", "pid": 154004, "rss_mb": 138.902, "state": "S (sleeping)", "threads": 106}, "restart_count": 922, "restart_count_per_hour": 9, "rtsp_timeout_count": 512, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 923}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.879, "sample_count": 2372, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1972081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:09:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.226, "last_frame_pushed_at": "2026-07-06T08:09:08+09:00", "last_frame_read_age_sec": 4.336, "last_frame_read_at": "2026-07-06T08:09:08+09:00", "mjpeg": {"age_sec": 4.208, "exists": true, "mtime": "2026-07-06T08:09:09.015526+09:00", "mtime_epoch": 1783292949.0155258, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201915}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:09:13+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9566.527, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.139, "pushed": 2.139, "read": 2.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.884, "count": 64, "max_ms": 37.578, "p95_ms": 29.295}, "gstreamer_push": {"avg_ms": 1.167, "count": 64, "max_ms": 1.403, "p95_ms": 1.264}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.795, "count": 64, "max_ms": 70.39, "p95_ms": 62.349}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.375, "count": 64, "max_ms": 81.113, "p95_ms": 77.123}, "yolo11_640_fire_11class": {"avg_ms": 48.841, "count": 64, "max_ms": 64.427, "p95_ms": 61.359}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.242, "count": 64, "max_ms": 76.608, "p95_ms": 75.371}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.479, "count": 64, "max_ms": 114.343, "p95_ms": 110.587}, "read_interval": {"avg_ms": 407.475, "count": 63, "max_ms": 10131.645, "p95_ms": 127.72}, "triton_wait": {"avg_ms": 57.79, "count": 64, "max_ms": 81.125, "p95_ms": 77.135}, "visualize": {"avg_ms": 3.333, "count": 64, "max_ms": 4.745, "p95_ms": 4.246}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20599}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16438.867, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ab1641f000] error while decoding MB 9 44, bytestream -13 [in#0/rtsp @ 0x55ab163d7e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55ab163d7e00] Error retrieving a packet from demuxer: Input/output error", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 147, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:14:49+09:00", "last_error_summary": "[h264 @ 0x55ab1641f000] error while decoding MB 9 44, bytestream -13 [in#0/rtsp @ 0x55ab163d7e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55ab163d7e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-06T07:59:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 904.287, "last_started_at": "2026-07-06T07:59:44+09:00", "pid": 153745, "process": {"exists": false, "pid": 153745}, "restart_count": 148, "restart_count_per_hour": 2, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.271, "sample_count": 2763, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2471898, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:14:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T08:14:49+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T08:14:49+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-06T08:14:49.078948+09:00", "mtime_epoch": 1783293289.0789475, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 204403}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T08:14:49+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2557.754, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 32, "pushed_frames": 32, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.819, "pushed": 1.819, "read": 1.876}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.622, "count": 32, "max_ms": 10.276, "p95_ms": 9.262}, "gstreamer_push": {"avg_ms": 1.296, "count": 32, "max_ms": 1.458, "p95_ms": 1.432}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.961, "count": 32, "max_ms": 65.924, "p95_ms": 63.942}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.547, "count": 32, "max_ms": 78.012, "p95_ms": 76.599}, "yolo11_640_fire_11class": {"avg_ms": 49.798, "count": 32, "max_ms": 63.809, "p95_ms": 62.155}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.638, "count": 32, "max_ms": 75.275, "p95_ms": 75.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.559, "count": 32, "max_ms": 95.019, "p95_ms": 93.87}, "read_interval": {"avg_ms": 1077.859, "count": 33, "max_ms": 18037.87, "p95_ms": 14722.893}, "triton_wait": {"avg_ms": 58.226, "count": 32, "max_ms": 78.024, "p95_ms": 76.61}, "visualize": {"avg_ms": 3.712, "count": 32, "max_ms": 4.431, "p95_ms": 4.327}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.593, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20599}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16438.527, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:14:49+09:00", "last_error_summary": "[h264 @ 0x55ab1641f000] error while decoding MB 9 44, bytestream -13 [in#0/rtsp @ 0x55ab163d7e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55ab163d7e00] Error retrieving a packet from demuxer: Input/output error", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 147, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:14:49+09:00", "last_error_summary": "[h264 @ 0x55ab1641f000] error while decoding MB 9 44, bytestream -13 [in#0/rtsp @ 0x55ab163d7e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55ab163d7e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-06T08:14:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 904.323, "last_started_at": "2026-07-06T07:59:44+09:00", "pid": 153745, "process": {"exists": false, "pid": 153745}, "restart_count": 148, "restart_count_per_hour": 2, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.83, "sample_count": 2763, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2471899, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:14:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T08:14:49+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-06T08:14:49+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-06T08:14:49.078948+09:00", "mtime_epoch": 1783293289.0789475, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 204403}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T08:14:49+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2557.754, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.872, "pushed": 1.872, "read": 1.872}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.62, "count": 33, "max_ms": 10.276, "p95_ms": 9.262}, "gstreamer_push": {"avg_ms": 1.294, "count": 33, "max_ms": 1.458, "p95_ms": 1.432}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.407, "count": 33, "max_ms": 65.924, "p95_ms": 65.688}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.431, "count": 33, "max_ms": 78.012, "p95_ms": 76.599}, "yolo11_640_fire_11class": {"avg_ms": 50.251, "count": 33, "max_ms": 64.764, "p95_ms": 63.809}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.166, "count": 33, "max_ms": 75.275, "p95_ms": 75.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.971, "count": 33, "max_ms": 95.019, "p95_ms": 93.87}, "read_interval": {"avg_ms": 1077.859, "count": 33, "max_ms": 18037.87, "p95_ms": 14722.893}, "triton_wait": {"avg_ms": 58.645, "count": 33, "max_ms": 78.024, "p95_ms": 76.61}, "visualize": {"avg_ms": 3.709, "count": 33, "max_ms": 4.431, "p95_ms": 4.327}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.629, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20599}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16438.52, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 147, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:14:49+09:00", "last_error_summary": "[h264 @ 0x55ab1641f000] error while decoding MB 9 44, bytestream -13 [in#0/rtsp @ 0x55ab163d7e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55ab163d7e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-06T08:14:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 904.331, "last_started_at": "2026-07-06T07:59:44+09:00", "pid": 153745, "process": {"exists": false, "pid": 153745}, "restart_count": 148, "restart_count_per_hour": 2, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.83, "sample_count": 2763, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2471899, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:14:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T08:14:49+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-06T08:14:49+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-06T08:14:49.078948+09:00", "mtime_epoch": 1783293289.0789475, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 204403}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T08:14:49+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2557.754, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.871, "pushed": 1.871, "read": 1.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.62, "count": 33, "max_ms": 10.276, "p95_ms": 9.262}, "gstreamer_push": {"avg_ms": 1.294, "count": 33, "max_ms": 1.458, "p95_ms": 1.432}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.407, "count": 33, "max_ms": 65.924, "p95_ms": 65.688}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.431, "count": 33, "max_ms": 78.012, "p95_ms": 76.599}, "yolo11_640_fire_11class": {"avg_ms": 50.251, "count": 33, "max_ms": 64.764, "p95_ms": 63.809}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.166, "count": 33, "max_ms": 75.275, "p95_ms": 75.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.971, "count": 33, "max_ms": 95.019, "p95_ms": 93.87}, "read_interval": {"avg_ms": 1077.859, "count": 33, "max_ms": 18037.87, "p95_ms": 14722.893}, "triton_wait": {"avg_ms": 58.645, "count": 33, "max_ms": 78.024, "p95_ms": 76.61}, "visualize": {"avg_ms": 3.709, "count": 33, "max_ms": 4.431, "p95_ms": 4.327}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.636, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20599}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16456.133, "limit_mb": 8796093022207.996}, "detail": {"pid": 154111, "restart_count": 149, "start_count": 150}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 147, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:14:49+09:00", "last_error_summary": "[h264 @ 0x55ab1641f000] error while decoding MB 9 44, bytestream -13 [in#0/rtsp @ 0x55ab163d7e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55ab163d7e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-06T08:14:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:14:51+09:00", "pid": 154111, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.773, "name": "ffmpeg", "pid": 154111, "rss_mb": 52.773, "state": "R (running)", "threads": 5}, "restart_count": 149, "restart_count_per_hour": 3, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 129.938, "sample_count": 2740, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2471899, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:14:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.342, "last_frame_pushed_at": "2026-07-06T08:14:49+09:00", "last_frame_read_age_sec": 4.434, "last_frame_read_at": "2026-07-06T08:14:49+09:00", "mjpeg": {"age_sec": 4.325, "exists": true, "mtime": "2026-07-06T08:14:49.178950+09:00", "mtime_epoch": 1783293289.1789503, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 204325}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T08:14:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2557.754, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 33, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.502, "pushed": 1.502, "read": 1.502}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.62, "count": 33, "max_ms": 10.276, "p95_ms": 9.262}, "gstreamer_push": {"avg_ms": 1.294, "count": 33, "max_ms": 1.458, "p95_ms": 1.432}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.407, "count": 33, "max_ms": 65.924, "p95_ms": 65.688}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.431, "count": 33, "max_ms": 78.012, "p95_ms": 76.599}, "yolo11_640_fire_11class": {"avg_ms": 50.251, "count": 33, "max_ms": 64.764, "p95_ms": 63.809}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.166, "count": 33, "max_ms": 75.275, "p95_ms": 75.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.971, "count": 33, "max_ms": 95.019, "p95_ms": 93.87}, "read_interval": {"avg_ms": 1077.859, "count": 33, "max_ms": 18037.87, "p95_ms": 14722.893}, "triton_wait": {"avg_ms": 58.645, "count": 33, "max_ms": 78.024, "p95_ms": 76.61}, "visualize": {"avg_ms": 3.709, "count": 33, "max_ms": 4.431, "p95_ms": 4.327}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.965, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20615}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16454.766, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563979c8f580] error while decoding MB 31 55, bytestream -11 [in#0/rtsp @ 0x563979b15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563979b15e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:18:52+09:00", "last_error_summary": "[h264 @ 0x563979c8f580] error while decoding MB 31 55, bytestream -11 [in#0/rtsp @ 0x563979b15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563979b15e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:14:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.828, "last_started_at": "2026-07-06T08:14:51+09:00", "pid": 154111, "process": {"exists": false, "pid": 154111}, "restart_count": 149, "restart_count_per_hour": 3, "rtsp_timeout_count": 254, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.474, "sample_count": 2225, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2473858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:18:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-06T08:18:52+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-06T08:18:52+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-06T08:18:52.269421+09:00", "mtime_epoch": 1783293532.269421, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242552}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T08:18:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2558.785, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.676, "pushed": 0.676, "read": 0.732}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.255, "count": 13, "max_ms": 11.244, "p95_ms": 11.244}, "gstreamer_push": {"avg_ms": 1.13, "count": 12, "max_ms": 1.255, "p95_ms": 1.255}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.887, "count": 13, "max_ms": 51.296, "p95_ms": 51.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.746, "count": 13, "max_ms": 66.437, "p95_ms": 66.437}, "yolo11_640_fire_11class": {"avg_ms": 44.113, "count": 13, "max_ms": 52.833, "p95_ms": 52.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.071, "count": 13, "max_ms": 66.998, "p95_ms": 66.998}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.364, "count": 12, "max_ms": 120.637, "p95_ms": 120.637}, "read_interval": {"avg_ms": 2487.872, "count": 13, "max_ms": 16477.207, "p95_ms": 16477.207}, "triton_wait": {"avg_ms": 51.187, "count": 13, "max_ms": 67.013, "p95_ms": 67.013}, "visualize": {"avg_ms": 4.608, "count": 13, "max_ms": 5.079, "p95_ms": 5.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.749, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20615}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16454.281, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:18:52+09:00", "last_error_summary": "[h264 @ 0x563979c8f580] error while decoding MB 31 55, bytestream -11 [in#0/rtsp @ 0x563979b15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563979b15e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:18:52+09:00", "last_error_summary": "[h264 @ 0x563979c8f580] error while decoding MB 31 55, bytestream -11 [in#0/rtsp @ 0x563979b15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563979b15e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:18:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.873, "last_started_at": "2026-07-06T08:14:51+09:00", "pid": 154111, "process": {"exists": false, "pid": 154111}, "restart_count": 149, "restart_count_per_hour": 3, "rtsp_timeout_count": 254, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.474, "sample_count": 2226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2473858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:18:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T08:18:52+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-06T08:18:52+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T08:18:52.269421+09:00", "mtime_epoch": 1783293532.269421, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242552}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T08:18:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2558.785, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.255, "count": 13, "max_ms": 11.244, "p95_ms": 11.244}, "gstreamer_push": {"avg_ms": 1.131, "count": 13, "max_ms": 1.255, "p95_ms": 1.255}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.887, "count": 13, "max_ms": 51.296, "p95_ms": 51.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.746, "count": 13, "max_ms": 66.437, "p95_ms": 66.437}, "yolo11_640_fire_11class": {"avg_ms": 44.113, "count": 13, "max_ms": 52.833, "p95_ms": 52.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.071, "count": 13, "max_ms": 66.998, "p95_ms": 66.998}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.155, "count": 13, "max_ms": 120.637, "p95_ms": 120.637}, "read_interval": {"avg_ms": 2487.872, "count": 13, "max_ms": 16477.207, "p95_ms": 16477.207}, "triton_wait": {"avg_ms": 51.187, "count": 13, "max_ms": 67.013, "p95_ms": 67.013}, "visualize": {"avg_ms": 4.608, "count": 13, "max_ms": 5.079, "p95_ms": 5.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.794, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20615}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16454.262, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:18:52+09:00", "last_error_summary": "[h264 @ 0x563979c8f580] error while decoding MB 31 55, bytestream -11 [in#0/rtsp @ 0x563979b15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563979b15e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:18:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.879, "last_started_at": "2026-07-06T08:14:51+09:00", "pid": 154111, "process": {"exists": false, "pid": 154111}, "restart_count": 149, "restart_count_per_hour": 3, "rtsp_timeout_count": 254, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.474, "sample_count": 2226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2473858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:18:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T08:18:52+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T08:18:52+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T08:18:52.269421+09:00", "mtime_epoch": 1783293532.269421, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242552}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T08:18:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2558.785, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.73, "pushed": 0.73, "read": 0.73}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.255, "count": 13, "max_ms": 11.244, "p95_ms": 11.244}, "gstreamer_push": {"avg_ms": 1.131, "count": 13, "max_ms": 1.255, "p95_ms": 1.255}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.887, "count": 13, "max_ms": 51.296, "p95_ms": 51.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.746, "count": 13, "max_ms": 66.437, "p95_ms": 66.437}, "yolo11_640_fire_11class": {"avg_ms": 44.113, "count": 13, "max_ms": 52.833, "p95_ms": 52.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.071, "count": 13, "max_ms": 66.998, "p95_ms": 66.998}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.155, "count": 13, "max_ms": 120.637, "p95_ms": 120.637}, "read_interval": {"avg_ms": 2487.872, "count": 13, "max_ms": 16477.207, "p95_ms": 16477.207}, "triton_wait": {"avg_ms": 51.187, "count": 13, "max_ms": 67.013, "p95_ms": 67.013}, "visualize": {"avg_ms": 4.608, "count": 13, "max_ms": 5.079, "p95_ms": 5.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.8, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20615}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16474.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 154263, "restart_count": 150, "start_count": 151}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:18:52+09:00", "last_error_summary": "[h264 @ 0x563979c8f580] error while decoding MB 31 55, bytestream -11 [in#0/rtsp @ 0x563979b15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563979b15e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:18:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:18:55+09:00", "pid": 154263, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.676, "name": "ffmpeg", "pid": 154263, "rss_mb": 57.676, "state": "S (sleeping)", "threads": 1}, "restart_count": 150, "restart_count_per_hour": 4, "rtsp_timeout_count": 254, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.753, "sample_count": 2173, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2473858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:18:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.733, "last_frame_pushed_at": "2026-07-06T08:18:52+09:00", "last_frame_read_age_sec": 4.841, "last_frame_read_at": "2026-07-06T08:18:52+09:00", "mjpeg": {"age_sec": 4.715, "exists": true, "mtime": "2026-07-06T08:18:52.389424+09:00", "mtime_epoch": 1783293532.389424, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242427}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T08:18:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2558.785, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.577, "pushed": 0.577, "read": 0.577}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.255, "count": 13, "max_ms": 11.244, "p95_ms": 11.244}, "gstreamer_push": {"avg_ms": 1.131, "count": 13, "max_ms": 1.255, "p95_ms": 1.255}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.887, "count": 13, "max_ms": 51.296, "p95_ms": 51.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.746, "count": 13, "max_ms": 66.437, "p95_ms": 66.437}, "yolo11_640_fire_11class": {"avg_ms": 44.113, "count": 13, "max_ms": 52.833, "p95_ms": 52.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.071, "count": 13, "max_ms": 66.998, "p95_ms": 66.998}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.155, "count": 13, "max_ms": 120.637, "p95_ms": 120.637}, "read_interval": {"avg_ms": 2487.872, "count": 13, "max_ms": 16477.207, "p95_ms": 16477.207}, "triton_wait": {"avg_ms": 51.187, "count": 13, "max_ms": 67.013, "p95_ms": 67.013}, "visualize": {"avg_ms": 4.608, "count": 13, "max_ms": 5.079, "p95_ms": 5.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.522, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16487.98, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5602e5c09180] left block unavailable for requested intra mode [h264 @ 0x5602e5c09180] error while decoding MB 0 23, bytestream 43 [h264 @ 0x5602e5b7be00] cabac decode of qscale diff failed at 10 41 [h264 @ 0x5602e5b7be00] error while decoding MB 10 41, bytestream 101688 [h264 @ 0x5602e5c09180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5602e5c09180] error while decoding MB 0 35, bytestream 74359 [h264 @ 0x5602e6699e40] left block unavailable for requested intra4x4 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 830, "h264_decode_error_count": 2018, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:21:33+09:00", "last_error_summary": "[h264 @ 0x5602e5c09180] left block unavailable for requested intra mode [h264 @ 0x5602e5c09180] error while decoding MB 0 23, bytestream 43 [h264 @ 0x5602e5b7be00] cabac decode of qscale diff failed at 10 41 [h264 @ 0x5602e5b7be00] error while decoding MB 10 41, bytestream 101688 [h264 @ 0x5602e5c09180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5602e5c09180] error while decoding MB 0 35, bytestream 74359 [h264 @ 0x5602e6699e40] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-06T08:09:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 742.083, "last_started_at": "2026-07-06T08:09:11+09:00", "pid": 154004, "process": {"exists": false, "pid": 154004}, "restart_count": 922, "restart_count_per_hour": 8, "rtsp_timeout_count": 514, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 923}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.543, "sample_count": 2634, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1978831, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:21:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.089, "last_frame_pushed_at": "2026-07-06T08:21:33+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-06T08:21:33+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-06T08:21:33.233208+09:00", "mtime_epoch": 1783293693.2332084, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209059}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:21:33+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9569.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.342, "pushed": 3.342, "read": 3.342}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.885, "count": 100, "max_ms": 36.162, "p95_ms": 31.834}, "gstreamer_push": {"avg_ms": 1.211, "count": 100, "max_ms": 1.611, "p95_ms": 1.333}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.609, "count": 100, "max_ms": 65.169, "p95_ms": 64.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.365, "count": 100, "max_ms": 84.629, "p95_ms": 77.44}, "yolo11_640_fire_11class": {"avg_ms": 49.927, "count": 100, "max_ms": 66.306, "p95_ms": 63.128}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.955, "count": 100, "max_ms": 78.025, "p95_ms": 74.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.021, "count": 100, "max_ms": 126.909, "p95_ms": 111.544}, "read_interval": {"avg_ms": 299.35, "count": 100, "max_ms": 19242.898, "p95_ms": 128.689}, "triton_wait": {"avg_ms": 58.228, "count": 100, "max_ms": 84.644, "p95_ms": 77.677}, "visualize": {"avg_ms": 3.295, "count": 100, "max_ms": 4.226, "p95_ms": 3.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16487.93, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:21:33+09:00", "last_error_summary": "[h264 @ 0x5602e5c09180] left block unavailable for requested intra mode [h264 @ 0x5602e5c09180] error while decoding MB 0 23, bytestream 43 [h264 @ 0x5602e5b7be00] cabac decode of qscale diff failed at 10 41 [h264 @ 0x5602e5b7be00] error while decoding MB 10 41, bytestream 101688 [h264 @ 0x5602e5c09180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5602e5c09180] error while decoding MB 0 35, bytestream 74359 [h264 @ 0x5602e6699e40] left block unavailable for requested intra4x4 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 830, "h264_decode_error_count": 2018, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:21:33+09:00", "last_error_summary": "[h264 @ 0x5602e5c09180] left block unavailable for requested intra mode [h264 @ 0x5602e5c09180] error while decoding MB 0 23, bytestream 43 [h264 @ 0x5602e5b7be00] cabac decode of qscale diff failed at 10 41 [h264 @ 0x5602e5b7be00] error while decoding MB 10 41, bytestream 101688 [h264 @ 0x5602e5c09180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5602e5c09180] error while decoding MB 0 35, bytestream 74359 [h264 @ 0x5602e6699e40] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-06T08:21:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 742.113, "last_started_at": "2026-07-06T08:09:11+09:00", "pid": 154004, "process": {"exists": false, "pid": 154004}, "restart_count": 922, "restart_count_per_hour": 8, "rtsp_timeout_count": 514, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 923}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.543, "sample_count": 2635, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1978832, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:21:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T08:21:33+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T08:21:33+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T08:21:33.233208+09:00", "mtime_epoch": 1783293693.2332084, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209059}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:21:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9569.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.372, "pushed": 3.372, "read": 3.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.894, "count": 101, "max_ms": 36.162, "p95_ms": 31.834}, "gstreamer_push": {"avg_ms": 1.213, "count": 101, "max_ms": 1.611, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.599, "count": 101, "max_ms": 65.169, "p95_ms": 64.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.364, "count": 101, "max_ms": 84.629, "p95_ms": 77.44}, "yolo11_640_fire_11class": {"avg_ms": 49.924, "count": 101, "max_ms": 66.306, "p95_ms": 63.128}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.921, "count": 101, "max_ms": 78.025, "p95_ms": 74.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.039, "count": 101, "max_ms": 126.909, "p95_ms": 111.544}, "read_interval": {"avg_ms": 299.35, "count": 100, "max_ms": 19242.898, "p95_ms": 128.689}, "triton_wait": {"avg_ms": 58.208, "count": 101, "max_ms": 84.644, "p95_ms": 77.677}, "visualize": {"avg_ms": 3.316, "count": 101, "max_ms": 5.41, "p95_ms": 3.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16487.969, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 830, "h264_decode_error_count": 2018, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:21:33+09:00", "last_error_summary": "[h264 @ 0x5602e5c09180] left block unavailable for requested intra mode [h264 @ 0x5602e5c09180] error while decoding MB 0 23, bytestream 43 [h264 @ 0x5602e5b7be00] cabac decode of qscale diff failed at 10 41 [h264 @ 0x5602e5b7be00] error while decoding MB 10 41, bytestream 101688 [h264 @ 0x5602e5c09180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5602e5c09180] error while decoding MB 0 35, bytestream 74359 [h264 @ 0x5602e6699e40] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-06T08:21:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 742.12, "last_started_at": "2026-07-06T08:09:11+09:00", "pid": 154004, "process": {"exists": false, "pid": 154004}, "restart_count": 922, "restart_count_per_hour": 8, "rtsp_timeout_count": 514, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 923}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.543, "sample_count": 2635, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1978832, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:21:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-06T08:21:33+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T08:21:33+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T08:21:33.233208+09:00", "mtime_epoch": 1783293693.2332084, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209059}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:21:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9569.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.371, "pushed": 3.371, "read": 3.338}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.894, "count": 101, "max_ms": 36.162, "p95_ms": 31.834}, "gstreamer_push": {"avg_ms": 1.213, "count": 101, "max_ms": 1.611, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.599, "count": 101, "max_ms": 65.169, "p95_ms": 64.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.364, "count": 101, "max_ms": 84.629, "p95_ms": 77.44}, "yolo11_640_fire_11class": {"avg_ms": 49.924, "count": 101, "max_ms": 66.306, "p95_ms": 63.128}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.921, "count": 101, "max_ms": 78.025, "p95_ms": 74.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.039, "count": 101, "max_ms": 126.909, "p95_ms": 111.544}, "read_interval": {"avg_ms": 299.35, "count": 100, "max_ms": 19242.898, "p95_ms": 128.689}, "triton_wait": {"avg_ms": 58.208, "count": 101, "max_ms": 84.644, "p95_ms": 77.677}, "visualize": {"avg_ms": 3.316, "count": 101, "max_ms": 5.41, "p95_ms": 3.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16592.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 154415, "restart_count": 923, "start_count": 924}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 830, "h264_decode_error_count": 2018, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:21:33+09:00", "last_error_summary": "[h264 @ 0x5602e5c09180] left block unavailable for requested intra mode [h264 @ 0x5602e5c09180] error while decoding MB 0 23, bytestream 43 [h264 @ 0x5602e5b7be00] cabac decode of qscale diff failed at 10 41 [h264 @ 0x5602e5b7be00] error while decoding MB 10 41, bytestream 101688 [h264 @ 0x5602e5c09180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5602e5c09180] error while decoding MB 0 35, bytestream 74359 [h264 @ 0x5602e6699e40] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-06T08:21:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:21:35+09:00", "pid": 154415, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.508, "name": "ffmpeg", "pid": 154415, "rss_mb": 136.508, "state": "S (sleeping)", "threads": 106}, "restart_count": 923, "restart_count_per_hour": 9, "rtsp_timeout_count": 514, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 924}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.812, "sample_count": 2596, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1978832, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:21:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.081, "last_frame_pushed_at": "2026-07-06T08:21:33+09:00", "last_frame_read_age_sec": 4.178, "last_frame_read_at": "2026-07-06T08:21:33+09:00", "mjpeg": {"age_sec": 4.065, "exists": true, "mtime": "2026-07-06T08:21:33.341211+09:00", "mtime_epoch": 1783293693.3412108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200201}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:21:37+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9569.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.104, "pushed": 2.104, "read": 2.07}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 27.265, "count": 63, "max_ms": 36.162, "p95_ms": 33.542}, "gstreamer_push": {"avg_ms": 1.225, "count": 63, "max_ms": 1.433, "p95_ms": 1.348}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.702, "count": 63, "max_ms": 65.169, "p95_ms": 64.39}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.606, "count": 63, "max_ms": 84.629, "p95_ms": 78.715}, "yolo11_640_fire_11class": {"avg_ms": 49.972, "count": 63, "max_ms": 66.306, "p95_ms": 63.128}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.908, "count": 63, "max_ms": 78.025, "p95_ms": 74.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.48, "count": 63, "max_ms": 126.909, "p95_ms": 113.289}, "read_interval": {"avg_ms": 417.062, "count": 62, "max_ms": 19242.898, "p95_ms": 129.791}, "triton_wait": {"avg_ms": 58.183, "count": 63, "max_ms": 84.644, "p95_ms": 78.726}, "visualize": {"avg_ms": 3.334, "count": 63, "max_ms": 5.41, "p95_ms": 3.563}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16512}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16492.402, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 831, "h264_decode_error_count": 2026, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:26:43+09:00", "last_error_summary": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "last_exit_at": "2026-07-06T08:21:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 308.139, "last_started_at": "2026-07-06T08:21:35+09:00", "pid": 154415, "process": {"exists": false, "pid": 154415}, "restart_count": 923, "restart_count_per_hour": 8, "rtsp_timeout_count": 516, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 924}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.652, "sample_count": 2585, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1981475, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:26:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.09, "last_frame_pushed_at": "2026-07-06T08:26:43+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-06T08:26:43+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-06T08:26:43.471803+09:00", "mtime_epoch": 1783294003.471803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185925}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:26:43+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9571.168, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.406, "pushed": 3.406, "read": 3.406}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.539, "count": 102, "max_ms": 35.156, "p95_ms": 31.987}, "gstreamer_push": {"avg_ms": 1.188, "count": 102, "max_ms": 2.079, "p95_ms": 1.29}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.46, "count": 102, "max_ms": 65.765, "p95_ms": 63.553}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.778, "count": 102, "max_ms": 85.855, "p95_ms": 68.858}, "yolo11_640_fire_11class": {"avg_ms": 49.834, "count": 102, "max_ms": 65.226, "p95_ms": 62.834}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.869, "count": 102, "max_ms": 75.563, "p95_ms": 72.083}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.211, "count": 102, "max_ms": 126.508, "p95_ms": 106.098}, "read_interval": {"avg_ms": 293.833, "count": 102, "max_ms": 10125.804, "p95_ms": 122.132}, "triton_wait": {"avg_ms": 57.822, "count": 102, "max_ms": 85.871, "p95_ms": 72.609}, "visualize": {"avg_ms": 3.259, "count": 102, "max_ms": 4.001, "p95_ms": 3.654}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16512}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16492.277, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:26:43+09:00", "last_error_summary": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 831, "h264_decode_error_count": 2026, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:26:43+09:00", "last_error_summary": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "last_exit_at": "2026-07-06T08:26:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 308.17, "last_started_at": "2026-07-06T08:21:35+09:00", "pid": 154415, "process": {"exists": false, "pid": 154415}, "restart_count": 923, "restart_count_per_hour": 8, "rtsp_timeout_count": 516, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 924}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.652, "sample_count": 2586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1981476, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:26:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T08:26:43+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T08:26:43+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-06T08:26:43.471803+09:00", "mtime_epoch": 1783294003.471803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185925}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:26:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9571.168, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.436, "pushed": 3.436, "read": 3.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.585, "count": 103, "max_ms": 35.156, "p95_ms": 31.987}, "gstreamer_push": {"avg_ms": 1.19, "count": 103, "max_ms": 2.079, "p95_ms": 1.295}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.42, "count": 103, "max_ms": 65.765, "p95_ms": 63.553}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.751, "count": 103, "max_ms": 85.855, "p95_ms": 68.858}, "yolo11_640_fire_11class": {"avg_ms": 49.79, "count": 103, "max_ms": 65.226, "p95_ms": 62.834}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.801, "count": 103, "max_ms": 75.563, "p95_ms": 72.083}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.243, "count": 103, "max_ms": 126.508, "p95_ms": 106.098}, "read_interval": {"avg_ms": 293.833, "count": 102, "max_ms": 10125.804, "p95_ms": 122.132}, "triton_wait": {"avg_ms": 57.775, "count": 103, "max_ms": 85.871, "p95_ms": 72.609}, "visualize": {"avg_ms": 3.279, "count": 103, "max_ms": 5.328, "p95_ms": 3.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16512}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16492.23, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 831, "h264_decode_error_count": 2026, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:26:43+09:00", "last_error_summary": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "last_exit_at": "2026-07-06T08:26:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 308.18, "last_started_at": "2026-07-06T08:21:35+09:00", "pid": 154415, "process": {"exists": false, "pid": 154415}, "restart_count": 923, "restart_count_per_hour": 8, "rtsp_timeout_count": 516, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 924}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.652, "sample_count": 2586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1981476, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:26:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-06T08:26:43+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T08:26:43+09:00", "mjpeg": {"age_sec": 0.005, "exists": true, "mtime": "2026-07-06T08:26:43.579805+09:00", "mtime_epoch": 1783294003.5798051, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185877}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:26:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9571.168, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.435, "pushed": 3.435, "read": 3.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.585, "count": 103, "max_ms": 35.156, "p95_ms": 31.987}, "gstreamer_push": {"avg_ms": 1.19, "count": 103, "max_ms": 2.079, "p95_ms": 1.295}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.42, "count": 103, "max_ms": 65.765, "p95_ms": 63.553}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.751, "count": 103, "max_ms": 85.855, "p95_ms": 68.858}, "yolo11_640_fire_11class": {"avg_ms": 49.79, "count": 103, "max_ms": 65.226, "p95_ms": 62.834}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.801, "count": 103, "max_ms": 75.563, "p95_ms": 72.083}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.243, "count": 103, "max_ms": 126.508, "p95_ms": 106.098}, "read_interval": {"avg_ms": 293.833, "count": 102, "max_ms": 10125.804, "p95_ms": 122.132}, "triton_wait": {"avg_ms": 57.775, "count": 103, "max_ms": 85.871, "p95_ms": 72.609}, "visualize": {"avg_ms": 3.279, "count": 103, "max_ms": 5.328, "p95_ms": 3.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16512}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16599.34, "limit_mb": 8796093022207.996}, "detail": {"pid": 154522, "restart_count": 924, "start_count": 925}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 831, "h264_decode_error_count": 2026, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:26:43+09:00", "last_error_summary": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "last_exit_at": "2026-07-06T08:26:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:26:46+09:00", "pid": 154522, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.973, "name": "ffmpeg", "pid": 154522, "rss_mb": 138.973, "state": "S (sleeping)", "threads": 106}, "restart_count": 924, "restart_count_per_hour": 9, "rtsp_timeout_count": 516, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 925}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.678, "sample_count": 2540, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1981476, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:26:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.773, "last_frame_pushed_at": "2026-07-06T08:26:43+09:00", "last_frame_read_age_sec": 4.871, "last_frame_read_at": "2026-07-06T08:26:43+09:00", "mjpeg": {"age_sec": 4.757, "exists": true, "mtime": "2026-07-06T08:26:43.579805+09:00", "mtime_epoch": 1783294003.5798051, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185877}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:26:48+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9571.168, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.935, "pushed": 1.935, "read": 1.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 27.078, "count": 58, "max_ms": 35.156, "p95_ms": 34.138}, "gstreamer_push": {"avg_ms": 1.206, "count": 58, "max_ms": 2.079, "p95_ms": 1.374}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.308, "count": 58, "max_ms": 65.765, "p95_ms": 64.511}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.345, "count": 58, "max_ms": 85.855, "p95_ms": 70.415}, "yolo11_640_fire_11class": {"avg_ms": 49.672, "count": 58, "max_ms": 65.226, "p95_ms": 63.362}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.616, "count": 58, "max_ms": 75.563, "p95_ms": 72.388}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.252, "count": 58, "max_ms": 126.508, "p95_ms": 106.876}, "read_interval": {"avg_ms": 442.413, "count": 57, "max_ms": 10125.804, "p95_ms": 140.536}, "triton_wait": {"avg_ms": 58.284, "count": 58, "max_ms": 85.871, "p95_ms": 73.221}, "visualize": {"avg_ms": 3.306, "count": 58, "max_ms": 5.328, "p95_ms": 3.966}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16531}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16495.477, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:26:43+09:00", "last_error_summary": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 831, "h264_decode_error_count": 2026, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:26:43+09:00", "last_error_summary": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "last_exit_at": "2026-07-06T08:31:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 279.658, "last_started_at": "2026-07-06T08:26:46+09:00", "pid": 154522, "process": {"exists": false, "pid": 154522}, "restart_count": 924, "restart_count_per_hour": 8, "rtsp_timeout_count": 516, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 925}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.48, "sample_count": 2264, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1983729, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:31:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T08:31:25+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-06T08:31:25+09:00", "mjpeg": {"age_sec": 0.152, "exists": true, "mtime": "2026-07-06T08:31:25.841306+09:00", "mtime_epoch": 1783294285.8413062, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204587}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:31:25+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9572.141, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.247, "pushed": 7.247, "read": 7.213}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.35, "count": 217, "max_ms": 35.126, "p95_ms": 30.859}, "gstreamer_push": {"avg_ms": 1.12, "count": 217, "max_ms": 1.797, "p95_ms": 1.217}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.596, "count": 217, "max_ms": 65.605, "p95_ms": 62.856}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.67, "count": 217, "max_ms": 82.907, "p95_ms": 76.525}, "yolo11_640_fire_11class": {"avg_ms": 49.727, "count": 217, "max_ms": 66.742, "p95_ms": 64.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.737, "count": 217, "max_ms": 79.849, "p95_ms": 77.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.842, "count": 217, "max_ms": 158.089, "p95_ms": 150.989}, "read_interval": {"avg_ms": 138.457, "count": 216, "max_ms": 173.023, "p95_ms": 163.052}, "triton_wait": {"avg_ms": 60.297, "count": 217, "max_ms": 82.919, "p95_ms": 79.257}, "visualize": {"avg_ms": 4.264, "count": 217, "max_ms": 5.597, "p95_ms": 4.824}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16531}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16495.461, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 831, "h264_decode_error_count": 2026, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:26:43+09:00", "last_error_summary": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "last_exit_at": "2026-07-06T08:31:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 279.665, "last_started_at": "2026-07-06T08:26:46+09:00", "pid": 154522, "process": {"exists": false, "pid": 154522}, "restart_count": 924, "restart_count_per_hour": 8, "rtsp_timeout_count": 516, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 925}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.48, "sample_count": 2264, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1983729, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:31:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T08:31:25+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-06T08:31:25+09:00", "mjpeg": {"age_sec": 0.159, "exists": true, "mtime": "2026-07-06T08:31:25.841306+09:00", "mtime_epoch": 1783294285.8413062, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204587}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:31:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9572.141, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.245, "pushed": 7.245, "read": 7.212}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.35, "count": 217, "max_ms": 35.126, "p95_ms": 30.859}, "gstreamer_push": {"avg_ms": 1.12, "count": 217, "max_ms": 1.797, "p95_ms": 1.217}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.596, "count": 217, "max_ms": 65.605, "p95_ms": 62.856}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.67, "count": 217, "max_ms": 82.907, "p95_ms": 76.525}, "yolo11_640_fire_11class": {"avg_ms": 49.727, "count": 217, "max_ms": 66.742, "p95_ms": 64.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.737, "count": 217, "max_ms": 79.849, "p95_ms": 77.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.842, "count": 217, "max_ms": 158.089, "p95_ms": 150.989}, "read_interval": {"avg_ms": 138.457, "count": 216, "max_ms": 173.023, "p95_ms": 163.052}, "triton_wait": {"avg_ms": 60.297, "count": 217, "max_ms": 82.919, "p95_ms": 79.257}, "visualize": {"avg_ms": 4.264, "count": 217, "max_ms": 5.597, "p95_ms": 4.824}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16531}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16602.617, "limit_mb": 8796093022207.996}, "detail": {"pid": 154629, "restart_count": 925, "start_count": 926}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 831, "h264_decode_error_count": 2026, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:26:43+09:00", "last_error_summary": "[h264 @ 0x5586aab9a680] left block unavailable for requested intra mode [h264 @ 0x5586aab9a680] error while decoding MB 0 17, bytestream 180 [h264 @ 0x5586aab65f80] error while decoding MB 65 33, bytestream -20 [h264 @ 0x5586aaa64e80] cabac decode of qscale diff failed at 48 41 [h264 @ 0x5586aaa64e80] error while decoding MB 48 41, bytestream 684 [h264 @ 0x5586ab805580] error while decoding MB 35 38, bytestream -20 [h264 @ 0x5586ab805580] cabac decode of qscale diff failed at 19 44 [h264 @ 0x558", "last_exit_at": "2026-07-06T08:31:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:31:28+09:00", "pid": 154629, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.438, "name": "ffmpeg", "pid": 154629, "rss_mb": 139.438, "state": "S (sleeping)", "threads": 106}, "restart_count": 925, "restart_count_per_hour": 9, "rtsp_timeout_count": 516, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 926}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.48, "sample_count": 2264, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1983729, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:31:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.424, "last_frame_pushed_at": "2026-07-06T08:31:25+09:00", "last_frame_read_age_sec": 4.577, "last_frame_read_at": "2026-07-06T08:31:25+09:00", "mjpeg": {"age_sec": 4.405, "exists": true, "mtime": "2026-07-06T08:31:26.005309+09:00", "mtime_epoch": 1783294286.005309, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204632}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:31:30+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9572.141, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 187, "pushed_frames": 187, "read_frames": 186, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.25, "pushed": 6.25, "read": 6.216}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.256, "count": 187, "max_ms": 35.126, "p95_ms": 30.849}, "gstreamer_push": {"avg_ms": 1.119, "count": 187, "max_ms": 1.6, "p95_ms": 1.217}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.49, "count": 187, "max_ms": 65.605, "p95_ms": 63.204}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.171, "count": 187, "max_ms": 82.907, "p95_ms": 74.231}, "yolo11_640_fire_11class": {"avg_ms": 49.652, "count": 187, "max_ms": 66.742, "p95_ms": 64.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.475, "count": 187, "max_ms": 79.849, "p95_ms": 77.109}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.411, "count": 187, "max_ms": 158.089, "p95_ms": 148.895}, "read_interval": {"avg_ms": 137.049, "count": 186, "max_ms": 173.023, "p95_ms": 160.848}, "triton_wait": {"avg_ms": 59.845, "count": 187, "max_ms": 82.919, "p95_ms": 78.098}, "visualize": {"avg_ms": 4.224, "count": 187, "max_ms": 5.597, "p95_ms": 4.61}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16574}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16504.539, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a53443abc0] reference picture missing during reorder [h264 @ 0x55a53443abc0] Missing reference picture, default is 65537 [h264 @ 0x55a53373d580] mmco: unref short failure [h264 @ 0x55a53373d580] illegal short term buffer state detected [h264 @ 0x55a533782c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a533782c80] error while decoding MB 0 44, bytestream 2224 [h264 @ 0x55a53443abc0] left block unavailable for requested intra mode [h264 @ 0x55a53443abc0] error wh", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 832, "h264_decode_error_count": 2032, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:41:12+09:00", "last_error_summary": "[h264 @ 0x55a53443abc0] reference picture missing during reorder [h264 @ 0x55a53443abc0] Missing reference picture, default is 65537 [h264 @ 0x55a53373d580] mmco: unref short failure [h264 @ 0x55a53373d580] illegal short term buffer state detected [h264 @ 0x55a533782c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a533782c80] error while decoding MB 0 44, bytestream 2224 [h264 @ 0x55a53443abc0] left block unavailable for requested intra mode [h264 @ 0x55a53443abc0] error wh", "last_exit_at": "2026-07-06T08:31:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 583.785, "last_started_at": "2026-07-06T08:31:28+09:00", "pid": 154629, "process": {"exists": false, "pid": 154629}, "restart_count": 925, "restart_count_per_hour": 8, "rtsp_timeout_count": 518, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 926}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.297, "sample_count": 2617, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1988977, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:41:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.088, "last_frame_pushed_at": "2026-07-06T08:41:12+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-06T08:41:12+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-06T08:41:12.120005+09:00", "mtime_epoch": 1783294872.120005, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 226115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:41:12+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9574.758, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.505, "pushed": 3.505, "read": 3.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.217, "count": 105, "max_ms": 30.86, "p95_ms": 28.876}, "gstreamer_push": {"avg_ms": 1.186, "count": 105, "max_ms": 2.135, "p95_ms": 1.29}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.678, "count": 105, "max_ms": 64.769, "p95_ms": 63.717}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.421, "count": 105, "max_ms": 79.025, "p95_ms": 63.382}, "yolo11_640_fire_11class": {"avg_ms": 50.171, "count": 105, "max_ms": 67.372, "p95_ms": 65.974}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.212, "count": 105, "max_ms": 74.956, "p95_ms": 71.309}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.195, "count": 105, "max_ms": 113.696, "p95_ms": 105.266}, "read_interval": {"avg_ms": 285.473, "count": 105, "max_ms": 10129.332, "p95_ms": 122.747}, "triton_wait": {"avg_ms": 55.041, "count": 105, "max_ms": 79.038, "p95_ms": 71.477}, "visualize": {"avg_ms": 3.282, "count": 105, "max_ms": 4.038, "p95_ms": 3.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16574}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16504.352, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:41:12+09:00", "last_error_summary": "[h264 @ 0x55a53443abc0] reference picture missing during reorder [h264 @ 0x55a53443abc0] Missing reference picture, default is 65537 [h264 @ 0x55a53373d580] mmco: unref short failure [h264 @ 0x55a53373d580] illegal short term buffer state detected [h264 @ 0x55a533782c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a533782c80] error while decoding MB 0 44, bytestream 2224 [h264 @ 0x55a53443abc0] left block unavailable for requested intra mode [h264 @ 0x55a53443abc0] error wh", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 832, "h264_decode_error_count": 2032, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:41:12+09:00", "last_error_summary": "[h264 @ 0x55a53443abc0] reference picture missing during reorder [h264 @ 0x55a53443abc0] Missing reference picture, default is 65537 [h264 @ 0x55a53373d580] mmco: unref short failure [h264 @ 0x55a53373d580] illegal short term buffer state detected [h264 @ 0x55a533782c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a533782c80] error while decoding MB 0 44, bytestream 2224 [h264 @ 0x55a53443abc0] left block unavailable for requested intra mode [h264 @ 0x55a53443abc0] error wh", "last_exit_at": "2026-07-06T08:41:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 583.81, "last_started_at": "2026-07-06T08:31:28+09:00", "pid": 154629, "process": {"exists": false, "pid": 154629}, "restart_count": 925, "restart_count_per_hour": 8, "rtsp_timeout_count": 518, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 926}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.297, "sample_count": 2618, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1988978, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:41:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T08:41:12+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-06T08:41:12+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-06T08:41:12.120005+09:00", "mtime_epoch": 1783294872.120005, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 226115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:41:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9574.758, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.535, "pushed": 3.535, "read": 3.502}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.239, "count": 106, "max_ms": 30.86, "p95_ms": 28.876}, "gstreamer_push": {"avg_ms": 1.189, "count": 106, "max_ms": 2.135, "p95_ms": 1.302}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.645, "count": 106, "max_ms": 64.769, "p95_ms": 63.717}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.432, "count": 106, "max_ms": 79.025, "p95_ms": 63.382}, "yolo11_640_fire_11class": {"avg_ms": 50.127, "count": 106, "max_ms": 67.372, "p95_ms": 65.974}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.161, "count": 106, "max_ms": 74.956, "p95_ms": 71.309}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.227, "count": 106, "max_ms": 113.696, "p95_ms": 105.266}, "read_interval": {"avg_ms": 285.473, "count": 105, "max_ms": 10129.332, "p95_ms": 122.747}, "triton_wait": {"avg_ms": 55.028, "count": 106, "max_ms": 79.038, "p95_ms": 71.477}, "visualize": {"avg_ms": 3.301, "count": 106, "max_ms": 5.261, "p95_ms": 3.552}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16574}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16504.148, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 832, "h264_decode_error_count": 2032, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:41:12+09:00", "last_error_summary": "[h264 @ 0x55a53443abc0] reference picture missing during reorder [h264 @ 0x55a53443abc0] Missing reference picture, default is 65537 [h264 @ 0x55a53373d580] mmco: unref short failure [h264 @ 0x55a53373d580] illegal short term buffer state detected [h264 @ 0x55a533782c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a533782c80] error while decoding MB 0 44, bytestream 2224 [h264 @ 0x55a53443abc0] left block unavailable for requested intra mode [h264 @ 0x55a53443abc0] error wh", "last_exit_at": "2026-07-06T08:41:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 583.818, "last_started_at": "2026-07-06T08:31:28+09:00", "pid": 154629, "process": {"exists": false, "pid": 154629}, "restart_count": 925, "restart_count_per_hour": 8, "rtsp_timeout_count": 518, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 926}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.297, "sample_count": 2617, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1988978, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:41:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T08:41:12+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T08:41:12+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T08:41:12.120005+09:00", "mtime_epoch": 1783294872.120005, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 226115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:41:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9574.758, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.534, "pushed": 3.534, "read": 3.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.239, "count": 106, "max_ms": 30.86, "p95_ms": 28.876}, "gstreamer_push": {"avg_ms": 1.189, "count": 106, "max_ms": 2.135, "p95_ms": 1.302}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.645, "count": 106, "max_ms": 64.769, "p95_ms": 63.717}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.432, "count": 106, "max_ms": 79.025, "p95_ms": 63.382}, "yolo11_640_fire_11class": {"avg_ms": 50.127, "count": 106, "max_ms": 67.372, "p95_ms": 65.974}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.161, "count": 106, "max_ms": 74.956, "p95_ms": 71.309}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.227, "count": 106, "max_ms": 113.696, "p95_ms": 105.266}, "read_interval": {"avg_ms": 285.473, "count": 105, "max_ms": 10129.332, "p95_ms": 122.747}, "triton_wait": {"avg_ms": 55.028, "count": 106, "max_ms": 79.038, "p95_ms": 71.477}, "visualize": {"avg_ms": 3.301, "count": 106, "max_ms": 5.261, "p95_ms": 3.552}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16574}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16611.594, "limit_mb": 8796093022207.996}, "detail": {"pid": 154736, "restart_count": 926, "start_count": 927}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 832, "h264_decode_error_count": 2032, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:41:12+09:00", "last_error_summary": "[h264 @ 0x55a53443abc0] reference picture missing during reorder [h264 @ 0x55a53443abc0] Missing reference picture, default is 65537 [h264 @ 0x55a53373d580] mmco: unref short failure [h264 @ 0x55a53373d580] illegal short term buffer state detected [h264 @ 0x55a533782c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a533782c80] error while decoding MB 0 44, bytestream 2224 [h264 @ 0x55a53443abc0] left block unavailable for requested intra mode [h264 @ 0x55a53443abc0] error wh", "last_exit_at": "2026-07-06T08:41:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:41:14+09:00", "pid": 154736, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.09, "name": "ffmpeg", "pid": 154736, "rss_mb": 139.09, "state": "S (sleeping)", "threads": 106}, "restart_count": 926, "restart_count_per_hour": 9, "rtsp_timeout_count": 518, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 927}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.63, "sample_count": 2574, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1988978, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:41:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.608, "last_frame_pushed_at": "2026-07-06T08:41:12+09:00", "last_frame_read_age_sec": 4.703, "last_frame_read_at": "2026-07-06T08:41:12+09:00", "mjpeg": {"age_sec": 4.593, "exists": true, "mtime": "2026-07-06T08:41:12.228007+09:00", "mtime_epoch": 1783294872.2280068, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 226431}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:41:16+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9574.758, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.038, "pushed": 2.038, "read": 2.004}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.488, "count": 61, "max_ms": 30.86, "p95_ms": 28.876}, "gstreamer_push": {"avg_ms": 1.199, "count": 61, "max_ms": 2.135, "p95_ms": 1.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.942, "count": 61, "max_ms": 64.717, "p95_ms": 63.717}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.014, "count": 61, "max_ms": 79.025, "p95_ms": 64.282}, "yolo11_640_fire_11class": {"avg_ms": 50.406, "count": 61, "max_ms": 67.372, "p95_ms": 65.506}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.578, "count": 61, "max_ms": 74.956, "p95_ms": 71.107}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.015, "count": 61, "max_ms": 113.696, "p95_ms": 105.266}, "read_interval": {"avg_ms": 422.284, "count": 60, "max_ms": 10129.332, "p95_ms": 127.351}, "triton_wait": {"avg_ms": 55.526, "count": 61, "max_ms": 79.038, "p95_ms": 71.477}, "visualize": {"avg_ms": 3.34, "count": 61, "max_ms": 5.261, "p95_ms": 3.627}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16608}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16508.676, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a8c9856300] left block unavailable for requested intra mode [h264 @ 0x55a8c9856300] error while decoding MB 0 23, bytestream 999 [h264 @ 0x55a8ca3f6ec0] error while decoding MB 3 36, bytestream -21 [h264 @ 0x55a8ca4c5d40] left block unavailable for requested intra mode [h264 @ 0x55a8ca4c5d40] error while decoding MB 0 22, bytestream 5024 [h264 @ 0x55a8c9819dc0] cabac decode of qscale diff failed at 43 17 [h264 @ 0x55a8c9819dc0] error while decoding MB 43 17, bytestream 1503 [h264 @ 0", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 833, "h264_decode_error_count": 2040, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:48:49+09:00", "last_error_summary": "[h264 @ 0x55a8c9856300] left block unavailable for requested intra mode [h264 @ 0x55a8c9856300] error while decoding MB 0 23, bytestream 999 [h264 @ 0x55a8ca3f6ec0] error while decoding MB 3 36, bytestream -21 [h264 @ 0x55a8ca4c5d40] left block unavailable for requested intra mode [h264 @ 0x55a8ca4c5d40] error while decoding MB 0 22, bytestream 5024 [h264 @ 0x55a8c9819dc0] cabac decode of qscale diff failed at 43 17 [h264 @ 0x55a8c9819dc0] error while decoding MB 43 17, bytestream 1503 [h264 @ 0", "last_exit_at": "2026-07-06T08:41:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 454.845, "last_started_at": "2026-07-06T08:41:14+09:00", "pid": 154736, "process": {"exists": false, "pid": 154736}, "restart_count": 926, "restart_count_per_hour": 8, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 927}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.064, "sample_count": 2610, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1992994, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:48:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.085, "last_frame_pushed_at": "2026-07-06T08:48:49+09:00", "last_frame_read_age_sec": 0.073, "last_frame_read_at": "2026-07-06T08:48:49+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-06T08:48:49.596396+09:00", "mtime_epoch": 1783295329.5963962, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198855}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:48:49+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9576.562, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.241, "pushed": 3.241, "read": 3.241}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.58, "count": 97, "max_ms": 41.064, "p95_ms": 29.907}, "gstreamer_push": {"avg_ms": 1.182, "count": 97, "max_ms": 1.636, "p95_ms": 1.318}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.566, "count": 97, "max_ms": 71.642, "p95_ms": 64.895}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.814, "count": 97, "max_ms": 78.897, "p95_ms": 69.778}, "yolo11_640_fire_11class": {"avg_ms": 49.789, "count": 97, "max_ms": 74.029, "p95_ms": 66.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.918, "count": 97, "max_ms": 76.115, "p95_ms": 74.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.745, "count": 97, "max_ms": 124.799, "p95_ms": 108.475}, "read_interval": {"avg_ms": 308.647, "count": 97, "max_ms": 10125.218, "p95_ms": 132.032}, "triton_wait": {"avg_ms": 55.29, "count": 97, "max_ms": 78.909, "p95_ms": 75.071}, "visualize": {"avg_ms": 3.271, "count": 97, "max_ms": 3.948, "p95_ms": 3.668}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16608}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16508.363, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:48:49+09:00", "last_error_summary": "[h264 @ 0x55a8c9856300] left block unavailable for requested intra mode [h264 @ 0x55a8c9856300] error while decoding MB 0 23, bytestream 999 [h264 @ 0x55a8ca3f6ec0] error while decoding MB 3 36, bytestream -21 [h264 @ 0x55a8ca4c5d40] left block unavailable for requested intra mode [h264 @ 0x55a8ca4c5d40] error while decoding MB 0 22, bytestream 5024 [h264 @ 0x55a8c9819dc0] cabac decode of qscale diff failed at 43 17 [h264 @ 0x55a8c9819dc0] error while decoding MB 43 17, bytestream 1503 [h264 @ 0", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 833, "h264_decode_error_count": 2040, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:48:49+09:00", "last_error_summary": "[h264 @ 0x55a8c9856300] left block unavailable for requested intra mode [h264 @ 0x55a8c9856300] error while decoding MB 0 23, bytestream 999 [h264 @ 0x55a8ca3f6ec0] error while decoding MB 3 36, bytestream -21 [h264 @ 0x55a8ca4c5d40] left block unavailable for requested intra mode [h264 @ 0x55a8ca4c5d40] error while decoding MB 0 22, bytestream 5024 [h264 @ 0x55a8c9819dc0] cabac decode of qscale diff failed at 43 17 [h264 @ 0x55a8c9819dc0] error while decoding MB 43 17, bytestream 1503 [h264 @ 0", "last_exit_at": "2026-07-06T08:48:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 454.873, "last_started_at": "2026-07-06T08:41:14+09:00", "pid": 154736, "process": {"exists": false, "pid": 154736}, "restart_count": 926, "restart_count_per_hour": 8, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 927}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.064, "sample_count": 2611, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1992995, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:48:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T08:48:49+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-06T08:48:49+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-06T08:48:49.596396+09:00", "mtime_epoch": 1783295329.5963962, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198855}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:48:49+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9576.562, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.271, "pushed": 3.271, "read": 3.238}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.581, "count": 98, "max_ms": 41.064, "p95_ms": 29.907}, "gstreamer_push": {"avg_ms": 1.184, "count": 98, "max_ms": 1.636, "p95_ms": 1.335}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.548, "count": 98, "max_ms": 71.642, "p95_ms": 64.895}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.807, "count": 98, "max_ms": 78.897, "p95_ms": 69.778}, "yolo11_640_fire_11class": {"avg_ms": 49.762, "count": 98, "max_ms": 74.029, "p95_ms": 66.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.883, "count": 98, "max_ms": 76.115, "p95_ms": 74.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.735, "count": 98, "max_ms": 124.799, "p95_ms": 108.475}, "read_interval": {"avg_ms": 308.647, "count": 97, "max_ms": 10125.218, "p95_ms": 132.032}, "triton_wait": {"avg_ms": 55.257, "count": 98, "max_ms": 78.909, "p95_ms": 75.071}, "visualize": {"avg_ms": 3.281, "count": 98, "max_ms": 4.244, "p95_ms": 3.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16608}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16508.352, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 833, "h264_decode_error_count": 2040, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:48:49+09:00", "last_error_summary": "[h264 @ 0x55a8c9856300] left block unavailable for requested intra mode [h264 @ 0x55a8c9856300] error while decoding MB 0 23, bytestream 999 [h264 @ 0x55a8ca3f6ec0] error while decoding MB 3 36, bytestream -21 [h264 @ 0x55a8ca4c5d40] left block unavailable for requested intra mode [h264 @ 0x55a8ca4c5d40] error while decoding MB 0 22, bytestream 5024 [h264 @ 0x55a8c9819dc0] cabac decode of qscale diff failed at 43 17 [h264 @ 0x55a8c9819dc0] error while decoding MB 43 17, bytestream 1503 [h264 @ 0", "last_exit_at": "2026-07-06T08:48:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 454.88, "last_started_at": "2026-07-06T08:41:14+09:00", "pid": 154736, "process": {"exists": false, "pid": 154736}, "restart_count": 926, "restart_count_per_hour": 8, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 927}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.064, "sample_count": 2611, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1992995, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:48:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T08:48:49+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T08:48:49+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-06T08:48:49.596396+09:00", "mtime_epoch": 1783295329.5963962, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198855}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:48:49+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9576.562, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.27, "pushed": 3.27, "read": 3.237}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.581, "count": 98, "max_ms": 41.064, "p95_ms": 29.907}, "gstreamer_push": {"avg_ms": 1.184, "count": 98, "max_ms": 1.636, "p95_ms": 1.335}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.548, "count": 98, "max_ms": 71.642, "p95_ms": 64.895}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.807, "count": 98, "max_ms": 78.897, "p95_ms": 69.778}, "yolo11_640_fire_11class": {"avg_ms": 49.762, "count": 98, "max_ms": 74.029, "p95_ms": 66.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.883, "count": 98, "max_ms": 76.115, "p95_ms": 74.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.735, "count": 98, "max_ms": 124.799, "p95_ms": 108.475}, "read_interval": {"avg_ms": 308.647, "count": 97, "max_ms": 10125.218, "p95_ms": 132.032}, "triton_wait": {"avg_ms": 55.257, "count": 98, "max_ms": 78.909, "p95_ms": 75.071}, "visualize": {"avg_ms": 3.281, "count": 98, "max_ms": 4.244, "p95_ms": 3.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16608}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16626.84, "limit_mb": 8796093022207.996}, "detail": {"pid": 154843, "restart_count": 927, "start_count": 928}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 833, "h264_decode_error_count": 2040, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:48:49+09:00", "last_error_summary": "[h264 @ 0x55a8c9856300] left block unavailable for requested intra mode [h264 @ 0x55a8c9856300] error while decoding MB 0 23, bytestream 999 [h264 @ 0x55a8ca3f6ec0] error while decoding MB 3 36, bytestream -21 [h264 @ 0x55a8ca4c5d40] left block unavailable for requested intra mode [h264 @ 0x55a8ca4c5d40] error while decoding MB 0 22, bytestream 5024 [h264 @ 0x55a8c9819dc0] cabac decode of qscale diff failed at 43 17 [h264 @ 0x55a8c9819dc0] error while decoding MB 43 17, bytestream 1503 [h264 @ 0", "last_exit_at": "2026-07-06T08:48:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:48:51+09:00", "pid": 154843, "process": {"exists": true, "fd_count": 5, "hwm_mb": 150.145, "name": "ffmpeg", "pid": 154843, "rss_mb": 150.145, "state": "S (sleeping)", "threads": 106}, "restart_count": 927, "restart_count_per_hour": 9, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 928}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.888, "sample_count": 2582, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1992995, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:48:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.047, "last_frame_pushed_at": "2026-07-06T08:48:49+09:00", "last_frame_read_age_sec": 4.138, "last_frame_read_at": "2026-07-06T08:48:49+09:00", "mjpeg": {"age_sec": 4.03, "exists": true, "mtime": "2026-07-06T08:48:49.700398+09:00", "mtime_epoch": 1783295329.7003982, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198824}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:48:53+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9576.562, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.001, "pushed": 2.001, "read": 1.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 26.574, "count": 60, "max_ms": 41.064, "p95_ms": 28.476}, "gstreamer_push": {"avg_ms": 1.185, "count": 60, "max_ms": 1.636, "p95_ms": 1.318}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.947, "count": 60, "max_ms": 71.642, "p95_ms": 66.449}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.913, "count": 60, "max_ms": 78.897, "p95_ms": 69.778}, "yolo11_640_fire_11class": {"avg_ms": 50.258, "count": 60, "max_ms": 74.029, "p95_ms": 66.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.461, "count": 60, "max_ms": 75.346, "p95_ms": 74.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.079, "count": 60, "max_ms": 124.799, "p95_ms": 108.475}, "read_interval": {"avg_ms": 439.583, "count": 59, "max_ms": 10125.218, "p95_ms": 138.909}, "triton_wait": {"avg_ms": 55.601, "count": 60, "max_ms": 78.909, "p95_ms": 75.071}, "visualize": {"avg_ms": 3.277, "count": 60, "max_ms": 4.244, "p95_ms": 3.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16646}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16513.438, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b7d97eccc0] error while decoding MB 12 19, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 834, "h264_decode_error_count": 2041, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:57:20+09:00", "last_error_summary": "[h264 @ 0x55b7d97eccc0] error while decoding MB 12 19, bytestream -7", "last_exit_at": "2026-07-06T08:48:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 508.4, "last_started_at": "2026-07-06T08:48:51+09:00", "pid": 154843, "process": {"exists": false, "pid": 154843}, "restart_count": 927, "restart_count_per_hour": 8, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 928}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.407, "sample_count": 2700, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1997572, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:57:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.091, "last_frame_pushed_at": "2026-07-06T08:57:20+09:00", "last_frame_read_age_sec": 0.079, "last_frame_read_at": "2026-07-06T08:57:20+09:00", "mjpeg": {"age_sec": 0.076, "exists": true, "mtime": "2026-07-06T08:57:20.053587+09:00", "mtime_epoch": 1783295840.0535872, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:57:20+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9578.883, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 282, "pushed_frames": 282, "read_frames": 282, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.401, "pushed": 9.401, "read": 9.401}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.85, "count": 281, "max_ms": 34.42, "p95_ms": 30.295}, "gstreamer_push": {"avg_ms": 1.187, "count": 282, "max_ms": 1.725, "p95_ms": 1.304}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.102, "count": 281, "max_ms": 71.847, "p95_ms": 64.351}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.531, "count": 281, "max_ms": 84.14, "p95_ms": 69.672}, "yolo11_640_fire_11class": {"avg_ms": 50.771, "count": 281, "max_ms": 74.645, "p95_ms": 67.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.213, "count": 281, "max_ms": 78.117, "p95_ms": 73.817}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.671, "count": 282, "max_ms": 119.63, "p95_ms": 109.208}, "read_interval": {"avg_ms": 106.385, "count": 282, "max_ms": 134.244, "p95_ms": 124.31}, "triton_wait": {"avg_ms": 57.002, "count": 281, "max_ms": 84.153, "p95_ms": 76.248}, "visualize": {"avg_ms": 3.278, "count": 281, "max_ms": 4.417, "p95_ms": 3.621}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16646}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16513.102, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T08:57:20+09:00", "last_error_summary": "[h264 @ 0x55b7d97eccc0] error while decoding MB 12 19, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 834, "h264_decode_error_count": 2041, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:57:20+09:00", "last_error_summary": "[h264 @ 0x55b7d97eccc0] error while decoding MB 12 19, bytestream -7", "last_exit_at": "2026-07-06T08:57:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 508.435, "last_started_at": "2026-07-06T08:48:51+09:00", "pid": 154843, "process": {"exists": false, "pid": 154843}, "restart_count": 927, "restart_count_per_hour": 8, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 928}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.407, "sample_count": 2701, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1997573, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:57:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T08:57:20+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T08:57:20+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T08:57:20.053587+09:00", "mtime_epoch": 1783295840.0535872, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:57:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9578.883, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 282, "pushed_frames": 282, "read_frames": 281, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.42, "pushed": 9.42, "read": 9.386}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.874, "count": 282, "max_ms": 34.42, "p95_ms": 30.578}, "gstreamer_push": {"avg_ms": 1.188, "count": 282, "max_ms": 1.725, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.093, "count": 282, "max_ms": 71.847, "p95_ms": 64.351}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.537, "count": 282, "max_ms": 84.14, "p95_ms": 69.672}, "yolo11_640_fire_11class": {"avg_ms": 50.765, "count": 282, "max_ms": 74.645, "p95_ms": 67.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.216, "count": 282, "max_ms": 78.117, "p95_ms": 73.817}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.731, "count": 282, "max_ms": 119.63, "p95_ms": 109.208}, "read_interval": {"avg_ms": 106.416, "count": 281, "max_ms": 134.244, "p95_ms": 124.31}, "triton_wait": {"avg_ms": 56.998, "count": 282, "max_ms": 84.153, "p95_ms": 76.248}, "visualize": {"avg_ms": 3.286, "count": 282, "max_ms": 5.482, "p95_ms": 3.654}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16646}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16513.145, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 834, "h264_decode_error_count": 2041, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:57:20+09:00", "last_error_summary": "[h264 @ 0x55b7d97eccc0] error while decoding MB 12 19, bytestream -7", "last_exit_at": "2026-07-06T08:57:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 508.445, "last_started_at": "2026-07-06T08:48:51+09:00", "pid": 154843, "process": {"exists": false, "pid": 154843}, "restart_count": 927, "restart_count_per_hour": 8, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 928}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.407, "sample_count": 2701, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1997573, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:57:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-06T08:57:20+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-06T08:57:20+09:00", "mjpeg": {"age_sec": 0.004, "exists": true, "mtime": "2026-07-06T08:57:20.169589+09:00", "mtime_epoch": 1783295840.169589, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180388}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:57:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9578.883, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 282, "pushed_frames": 282, "read_frames": 281, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.417, "pushed": 9.417, "read": 9.384}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.874, "count": 282, "max_ms": 34.42, "p95_ms": 30.578}, "gstreamer_push": {"avg_ms": 1.188, "count": 282, "max_ms": 1.725, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.093, "count": 282, "max_ms": 71.847, "p95_ms": 64.351}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.537, "count": 282, "max_ms": 84.14, "p95_ms": 69.672}, "yolo11_640_fire_11class": {"avg_ms": 50.765, "count": 282, "max_ms": 74.645, "p95_ms": 67.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.216, "count": 282, "max_ms": 78.117, "p95_ms": 73.817}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.731, "count": 282, "max_ms": 119.63, "p95_ms": 109.208}, "read_interval": {"avg_ms": 106.416, "count": 281, "max_ms": 134.244, "p95_ms": 124.31}, "triton_wait": {"avg_ms": 56.998, "count": 282, "max_ms": 84.153, "p95_ms": 76.248}, "visualize": {"avg_ms": 3.286, "count": 282, "max_ms": 5.482, "p95_ms": 3.654}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16646}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16620.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 154950, "restart_count": 928, "start_count": 929}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 834, "h264_decode_error_count": 2041, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T08:57:20+09:00", "last_error_summary": "[h264 @ 0x55b7d97eccc0] error while decoding MB 12 19, bytestream -7", "last_exit_at": "2026-07-06T08:57:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T08:57:22+09:00", "pid": 154950, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.289, "name": "ffmpeg", "pid": 154950, "rss_mb": 139.289, "state": "S (sleeping)", "threads": 106}, "restart_count": 928, "restart_count_per_hour": 9, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 929}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 127.852, "sample_count": 2669, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1997573, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T08:57:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.5, "last_frame_pushed_at": "2026-07-06T08:57:20+09:00", "last_frame_read_age_sec": 4.602, "last_frame_read_at": "2026-07-06T08:57:20+09:00", "mjpeg": {"age_sec": 4.483, "exists": true, "mtime": "2026-07-06T08:57:20.169589+09:00", "mtime_epoch": 1783295840.169589, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180388}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T08:57:24+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9578.883, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 240, "pushed_frames": 240, "read_frames": 240, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.004, "pushed": 8.004, "read": 8.004}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.909, "count": 240, "max_ms": 34.42, "p95_ms": 30.839}, "gstreamer_push": {"avg_ms": 1.19, "count": 240, "max_ms": 1.725, "p95_ms": 1.304}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.958, "count": 240, "max_ms": 71.847, "p95_ms": 64.075}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.139, "count": 240, "max_ms": 83.524, "p95_ms": 67.339}, "yolo11_640_fire_11class": {"avg_ms": 50.549, "count": 240, "max_ms": 74.645, "p95_ms": 67.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.011, "count": 240, "max_ms": 78.117, "p95_ms": 73.986}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.49, "count": 240, "max_ms": 118.502, "p95_ms": 109.196}, "read_interval": {"avg_ms": 106.195, "count": 240, "max_ms": 132.568, "p95_ms": 123.506}, "triton_wait": {"avg_ms": 56.698, "count": 240, "max_ms": 83.536, "p95_ms": 75.264}, "visualize": {"avg_ms": 3.294, "count": 240, "max_ms": 5.482, "p95_ms": 3.736}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16469.246, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562da2162540] error while decoding MB 44 49, bytestream -13 [in#0/rtsp @ 0x562da1fc9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562da1fc9e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:03:06+09:00", "last_error_summary": "[h264 @ 0x562da2162540] error while decoding MB 44 49, bytestream -13 [in#0/rtsp @ 0x562da1fc9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562da1fc9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T08:18:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2651.282, "last_started_at": "2026-07-06T08:18:55+09:00", "pid": 154263, "process": {"exists": false, "pid": 154263}, "restart_count": 150, "restart_count_per_hour": 2, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.456, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2499764, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:03:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.001, "last_frame_pushed_at": "2026-07-06T09:03:06+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-06T09:03:06+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-06T09:03:06.323725+09:00", "mtime_epoch": 1783296186.3237252, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239509}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:03:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2571.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.112, "count": 14, "max_ms": 10.067, "p95_ms": 10.067}, "gstreamer_push": {"avg_ms": 1.249, "count": 14, "max_ms": 1.327, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.688, "count": 14, "max_ms": 57.305, "p95_ms": 57.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.336, "count": 14, "max_ms": 76.23, "p95_ms": 76.23}, "yolo11_640_fire_11class": {"avg_ms": 50.419, "count": 14, "max_ms": 60.994, "p95_ms": 60.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.936, "count": 14, "max_ms": 73.388, "p95_ms": 73.388}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.644, "count": 14, "max_ms": 94.013, "p95_ms": 94.013}, "read_interval": {"avg_ms": 2168.734, "count": 14, "max_ms": 16793.943, "p95_ms": 16793.943}, "triton_wait": {"avg_ms": 53.977, "count": 14, "max_ms": 76.243, "p95_ms": 76.243}, "visualize": {"avg_ms": 3.596, "count": 14, "max_ms": 3.903, "p95_ms": 3.903}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.893, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16469.137, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T09:03:06+09:00", "last_error_summary": "[h264 @ 0x562da2162540] error while decoding MB 44 49, bytestream -13 [in#0/rtsp @ 0x562da1fc9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562da1fc9e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:03:06+09:00", "last_error_summary": "[h264 @ 0x562da2162540] error while decoding MB 44 49, bytestream -13 [in#0/rtsp @ 0x562da1fc9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562da1fc9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:03:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2651.298, "last_started_at": "2026-07-06T08:18:55+09:00", "pid": 154263, "process": {"exists": false, "pid": 154263}, "restart_count": 150, "restart_count_per_hour": 2, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.456, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2499764, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:03:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T09:03:06+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-06T09:03:06+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-06T09:03:06.323725+09:00", "mtime_epoch": 1783296186.3237252, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239509}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:03:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2571.16, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.112, "count": 14, "max_ms": 10.067, "p95_ms": 10.067}, "gstreamer_push": {"avg_ms": 1.249, "count": 14, "max_ms": 1.327, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.688, "count": 14, "max_ms": 57.305, "p95_ms": 57.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.336, "count": 14, "max_ms": 76.23, "p95_ms": 76.23}, "yolo11_640_fire_11class": {"avg_ms": 50.419, "count": 14, "max_ms": 60.994, "p95_ms": 60.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.936, "count": 14, "max_ms": 73.388, "p95_ms": 73.388}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.644, "count": 14, "max_ms": 94.013, "p95_ms": 94.013}, "read_interval": {"avg_ms": 2168.734, "count": 14, "max_ms": 16793.943, "p95_ms": 16793.943}, "triton_wait": {"avg_ms": 53.977, "count": 14, "max_ms": 76.243, "p95_ms": 76.243}, "visualize": {"avg_ms": 3.596, "count": 14, "max_ms": 3.903, "p95_ms": 3.903}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.908, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16469.078, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:03:06+09:00", "last_error_summary": "[h264 @ 0x562da2162540] error while decoding MB 44 49, bytestream -13 [in#0/rtsp @ 0x562da1fc9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562da1fc9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:03:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2651.303, "last_started_at": "2026-07-06T08:18:55+09:00", "pid": 154263, "process": {"exists": false, "pid": 154263}, "restart_count": 150, "restart_count_per_hour": 2, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.461, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2499764, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:03:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.022, "last_frame_pushed_at": "2026-07-06T09:03:06+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-06T09:03:06+09:00", "mjpeg": {"age_sec": null, "exists": false, "mtime": null, "mtime_epoch": null, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": null}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:03:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2571.16, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.112, "count": 14, "max_ms": 10.067, "p95_ms": 10.067}, "gstreamer_push": {"avg_ms": 1.249, "count": 14, "max_ms": 1.327, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.688, "count": 14, "max_ms": 57.305, "p95_ms": 57.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.336, "count": 14, "max_ms": 76.23, "p95_ms": 76.23}, "yolo11_640_fire_11class": {"avg_ms": 50.419, "count": 14, "max_ms": 60.994, "p95_ms": 60.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.936, "count": 14, "max_ms": 73.388, "p95_ms": 73.388}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.644, "count": 14, "max_ms": 94.013, "p95_ms": 94.013}, "read_interval": {"avg_ms": 2168.734, "count": 14, "max_ms": 16793.943, "p95_ms": 16793.943}, "triton_wait": {"avg_ms": 53.977, "count": 14, "max_ms": 76.243, "p95_ms": 76.243}, "visualize": {"avg_ms": 3.596, "count": 14, "max_ms": 3.903, "p95_ms": 3.903}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.914, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16636.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 155057, "restart_count": 151, "start_count": 152}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:03:06+09:00", "last_error_summary": "[h264 @ 0x562da2162540] error while decoding MB 44 49, bytestream -13 [in#0/rtsp @ 0x562da1fc9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562da1fc9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:03:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:03:08+09:00", "pid": 155057, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.695, "name": "ffmpeg", "pid": 155057, "rss_mb": 197.695, "state": "S (sleeping)", "threads": 106}, "restart_count": 151, "restart_count_per_hour": 3, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.5, "sample_count": 2365, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2499764, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:03:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.39, "last_frame_pushed_at": "2026-07-06T09:03:06+09:00", "last_frame_read_age_sec": 4.461, "last_frame_read_at": "2026-07-06T09:03:06+09:00", "mjpeg": {"age_sec": 4.371, "exists": true, "mtime": "2026-07-06T09:03:06.403727+09:00", "mtime_epoch": 1783296186.4037266, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239534}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:03:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2571.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.628, "pushed": 0.628, "read": 0.628}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.112, "count": 14, "max_ms": 10.067, "p95_ms": 10.067}, "gstreamer_push": {"avg_ms": 1.249, "count": 14, "max_ms": 1.327, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.688, "count": 14, "max_ms": 57.305, "p95_ms": 57.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.336, "count": 14, "max_ms": 76.23, "p95_ms": 76.23}, "yolo11_640_fire_11class": {"avg_ms": 50.419, "count": 14, "max_ms": 60.994, "p95_ms": 60.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.936, "count": 14, "max_ms": 73.388, "p95_ms": 73.388}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.644, "count": 14, "max_ms": 94.013, "p95_ms": 94.013}, "read_interval": {"avg_ms": 2168.734, "count": 14, "max_ms": 16793.943, "p95_ms": 16793.943}, "triton_wait": {"avg_ms": 53.977, "count": 14, "max_ms": 76.243, "p95_ms": 76.243}, "visualize": {"avg_ms": 3.596, "count": 14, "max_ms": 3.903, "p95_ms": 3.903}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.282, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16679}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16516.051, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e3e2c06bc0] error while decoding MB 4 39, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 835, "h264_decode_error_count": 2042, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:04:54+09:00", "last_error_summary": "[h264 @ 0x55e3e2c06bc0] error while decoding MB 4 39, bytestream -7", "last_exit_at": "2026-07-06T08:57:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 452.157, "last_started_at": "2026-07-06T08:57:22+09:00", "pid": 154950, "process": {"exists": false, "pid": 154950}, "restart_count": 928, "restart_count_per_hour": 7, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 929}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.691, "sample_count": 2687, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2001587, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:04:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.089, "last_frame_pushed_at": "2026-07-06T09:04:54+09:00", "last_frame_read_age_sec": 0.074, "last_frame_read_at": "2026-07-06T09:04:54+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-06T09:04:54.733639+09:00", "mtime_epoch": 1783296294.7336388, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190340}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T09:04:54+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9580.688, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 212, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.077, "pushed": 7.077, "read": 7.077}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 27.121, "count": 212, "max_ms": 43.299, "p95_ms": 32.294}, "gstreamer_push": {"avg_ms": 1.2, "count": 212, "max_ms": 1.9, "p95_ms": 1.308}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.16, "count": 212, "max_ms": 67.789, "p95_ms": 63.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.66, "count": 212, "max_ms": 82.31, "p95_ms": 75.495}, "yolo11_640_fire_11class": {"avg_ms": 50.516, "count": 212, "max_ms": 69.76, "p95_ms": 67.47}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.95, "count": 212, "max_ms": 78.442, "p95_ms": 75.095}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.016, "count": 212, "max_ms": 122.262, "p95_ms": 111.926}, "read_interval": {"avg_ms": 141.383, "count": 212, "max_ms": 7400.398, "p95_ms": 126.493}, "triton_wait": {"avg_ms": 56.98, "count": 212, "max_ms": 82.323, "p95_ms": 76.788}, "visualize": {"avg_ms": 3.308, "count": 212, "max_ms": 4.416, "p95_ms": 3.774}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16679}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16515.887, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T09:04:54+09:00", "last_error_summary": "[h264 @ 0x55e3e2c06bc0] error while decoding MB 4 39, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 835, "h264_decode_error_count": 2042, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:04:54+09:00", "last_error_summary": "[h264 @ 0x55e3e2c06bc0] error while decoding MB 4 39, bytestream -7", "last_exit_at": "2026-07-06T09:04:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 452.186, "last_started_at": "2026-07-06T08:57:22+09:00", "pid": 154950, "process": {"exists": false, "pid": 154950}, "restart_count": 928, "restart_count_per_hour": 7, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 929}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.691, "sample_count": 2687, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2001588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:04:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T09:04:54+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-06T09:04:54+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-06T09:04:54.733639+09:00", "mtime_epoch": 1783296294.7336388, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190340}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T09:04:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9580.688, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 212, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.104, "pushed": 7.104, "read": 7.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 27.138, "count": 213, "max_ms": 43.299, "p95_ms": 32.294}, "gstreamer_push": {"avg_ms": 1.2, "count": 213, "max_ms": 1.9, "p95_ms": 1.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.148, "count": 213, "max_ms": 67.789, "p95_ms": 63.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.642, "count": 213, "max_ms": 82.31, "p95_ms": 75.495}, "yolo11_640_fire_11class": {"avg_ms": 50.496, "count": 213, "max_ms": 69.76, "p95_ms": 67.47}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.928, "count": 213, "max_ms": 78.442, "p95_ms": 75.095}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.021, "count": 213, "max_ms": 122.262, "p95_ms": 111.926}, "read_interval": {"avg_ms": 141.383, "count": 212, "max_ms": 7400.398, "p95_ms": 126.493}, "triton_wait": {"avg_ms": 56.95, "count": 213, "max_ms": 82.323, "p95_ms": 76.788}, "visualize": {"avg_ms": 3.321, "count": 213, "max_ms": 6.096, "p95_ms": 3.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16679}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16516.238, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 835, "h264_decode_error_count": 2042, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:04:54+09:00", "last_error_summary": "[h264 @ 0x55e3e2c06bc0] error while decoding MB 4 39, bytestream -7", "last_exit_at": "2026-07-06T09:04:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 452.195, "last_started_at": "2026-07-06T08:57:22+09:00", "pid": 154950, "process": {"exists": false, "pid": 154950}, "restart_count": 928, "restart_count_per_hour": 7, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 929}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.691, "sample_count": 2687, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2001588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:04:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T09:04:54+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T09:04:54+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T09:04:54.733639+09:00", "mtime_epoch": 1783296294.7336388, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190340}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T09:04:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9580.688, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 212, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.102, "pushed": 7.102, "read": 7.068}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 27.138, "count": 213, "max_ms": 43.299, "p95_ms": 32.294}, "gstreamer_push": {"avg_ms": 1.2, "count": 213, "max_ms": 1.9, "p95_ms": 1.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.148, "count": 213, "max_ms": 67.789, "p95_ms": 63.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.642, "count": 213, "max_ms": 82.31, "p95_ms": 75.495}, "yolo11_640_fire_11class": {"avg_ms": 50.496, "count": 213, "max_ms": 69.76, "p95_ms": 67.47}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.928, "count": 213, "max_ms": 78.442, "p95_ms": 75.095}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.021, "count": 213, "max_ms": 122.262, "p95_ms": 111.926}, "read_interval": {"avg_ms": 141.383, "count": 212, "max_ms": 7400.398, "p95_ms": 126.493}, "triton_wait": {"avg_ms": 56.95, "count": 213, "max_ms": 82.323, "p95_ms": 76.788}, "visualize": {"avg_ms": 3.321, "count": 213, "max_ms": 6.096, "p95_ms": 3.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16679}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 16623.594, "limit_mb": 8796093022207.996}, "detail": {"pid": 155209, "restart_count": 929, "start_count": 930}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 835, "h264_decode_error_count": 2042, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:04:54+09:00", "last_error_summary": "[h264 @ 0x55e3e2c06bc0] error while decoding MB 4 39, bytestream -7", "last_exit_at": "2026-07-06T09:04:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:04:57+09:00", "pid": 155209, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.871, "name": "ffmpeg", "pid": 155209, "rss_mb": 138.871, "state": "S (sleeping)", "threads": 106}, "restart_count": 929, "restart_count_per_hour": 8, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 930}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.709, "sample_count": 2641, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2001588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:04:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 4.986, "last_frame_pushed_at": "2026-07-06T09:04:54+09:00", "last_frame_read_age_sec": 5.081, "last_frame_read_at": "2026-07-06T09:04:54+09:00", "mjpeg": {"age_sec": 4.97, "exists": true, "mtime": "2026-07-06T09:04:54.845641+09:00", "mtime_epoch": 1783296294.8456407, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190359}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T09:04:59+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9580.688, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 166, "pushed_frames": 166, "read_frames": 165, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.535, "pushed": 5.535, "read": 5.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 27.206, "count": 166, "max_ms": 42.211, "p95_ms": 32.294}, "gstreamer_push": {"avg_ms": 1.202, "count": 166, "max_ms": 1.9, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.23, "count": 166, "max_ms": 67.789, "p95_ms": 63.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.921, "count": 166, "max_ms": 82.31, "p95_ms": 76.23}, "yolo11_640_fire_11class": {"avg_ms": 50.606, "count": 166, "max_ms": 69.76, "p95_ms": 67.291}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.08, "count": 166, "max_ms": 78.442, "p95_ms": 75.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.335, "count": 166, "max_ms": 122.262, "p95_ms": 112.409}, "read_interval": {"avg_ms": 151.504, "count": 165, "max_ms": 7400.398, "p95_ms": 127.814}, "triton_wait": {"avg_ms": 57.175, "count": 166, "max_ms": 82.323, "p95_ms": 77.212}, "visualize": {"avg_ms": 3.329, "count": 166, "max_ms": 6.096, "p95_ms": 3.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20843}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16454.414, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x565391648ac0] error while decoding MB 11 37, bytestream -9 [in#0/rtsp @ 0x5653914fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653914fbe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:06:25+09:00", "last_error_summary": "[h264 @ 0x565391648ac0] error while decoding MB 11 37, bytestream -9 [in#0/rtsp @ 0x5653914fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653914fbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:03:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.549, "last_started_at": "2026-07-06T09:03:08+09:00", "pid": 155057, "process": {"exists": false, "pid": 155057}, "restart_count": 151, "restart_count_per_hour": 3, "rtsp_timeout_count": 258, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.537, "sample_count": 2162, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2501273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:06:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-06T09:06:25+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T09:06:25+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-06T09:06:25.271235+09:00", "mtime_epoch": 1783296385.2712348, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234797}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:06:25+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2572.191, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 21, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.703, "pushed": 0.703, "read": 0.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.776, "count": 21, "max_ms": 15.623, "p95_ms": 14.325}, "gstreamer_push": {"avg_ms": 1.015, "count": 21, "max_ms": 1.219, "p95_ms": 1.116}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.058, "count": 21, "max_ms": 74.482, "p95_ms": 66.325}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.308, "count": 21, "max_ms": 81.277, "p95_ms": 79.516}, "yolo11_640_fire_11class": {"avg_ms": 54.628, "count": 21, "max_ms": 73.761, "p95_ms": 65.199}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.362, "count": 21, "max_ms": 84.097, "p95_ms": 79.023}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.968, "count": 21, "max_ms": 171.635, "p95_ms": 169.062}, "read_interval": {"avg_ms": 1427.691, "count": 21, "max_ms": 16269.687, "p95_ms": 10622.988}, "triton_wait": {"avg_ms": 63.664, "count": 21, "max_ms": 84.109, "p95_ms": 79.528}, "visualize": {"avg_ms": 4.888, "count": 21, "max_ms": 6.143, "p95_ms": 5.138}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20843}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16453.945, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T09:06:25+09:00", "last_error_summary": "[h264 @ 0x565391648ac0] error while decoding MB 11 37, bytestream -9 [in#0/rtsp @ 0x5653914fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653914fbe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:06:25+09:00", "last_error_summary": "[h264 @ 0x565391648ac0] error while decoding MB 11 37, bytestream -9 [in#0/rtsp @ 0x5653914fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653914fbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:06:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.638, "last_started_at": "2026-07-06T09:03:08+09:00", "pid": 155057, "process": {"exists": false, "pid": 155057}, "restart_count": 151, "restart_count_per_hour": 3, "rtsp_timeout_count": 258, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.537, "sample_count": 2163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2501274, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:06:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T09:06:25+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-06T09:06:25+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-06T09:06:25.271235+09:00", "mtime_epoch": 1783296385.2712348, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234797}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:06:25+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2572.191, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 22, "pushed_frames": 22, "read_frames": 21, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.734, "pushed": 0.734, "read": 0.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.782, "count": 21, "max_ms": 15.623, "p95_ms": 14.325}, "gstreamer_push": {"avg_ms": 1.01, "count": 22, "max_ms": 1.219, "p95_ms": 1.116}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.3, "count": 21, "max_ms": 74.482, "p95_ms": 66.325}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.191, "count": 21, "max_ms": 81.277, "p95_ms": 79.516}, "yolo11_640_fire_11class": {"avg_ms": 53.849, "count": 21, "max_ms": 73.761, "p95_ms": 63.665}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.242, "count": 21, "max_ms": 84.097, "p95_ms": 79.023}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.925, "count": 22, "max_ms": 171.635, "p95_ms": 169.062}, "read_interval": {"avg_ms": 1427.691, "count": 21, "max_ms": 16269.687, "p95_ms": 10622.988}, "triton_wait": {"avg_ms": 62.544, "count": 21, "max_ms": 84.109, "p95_ms": 79.528}, "visualize": {"avg_ms": 5.103, "count": 21, "max_ms": 9.172, "p95_ms": 6.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20843}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16453.891, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:06:25+09:00", "last_error_summary": "[h264 @ 0x565391648ac0] error while decoding MB 11 37, bytestream -9 [in#0/rtsp @ 0x5653914fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653914fbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:06:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.642, "last_started_at": "2026-07-06T09:03:08+09:00", "pid": 155057, "process": {"exists": false, "pid": 155057}, "restart_count": 151, "restart_count_per_hour": 3, "rtsp_timeout_count": 258, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.537, "sample_count": 2163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2501274, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:06:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T09:06:25+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-06T09:06:25+09:00", "mjpeg": {"age_sec": 0.145, "exists": true, "mtime": "2026-07-06T09:06:25.271235+09:00", "mtime_epoch": 1783296385.2712348, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234797}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:06:25+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2572.191, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 22, "pushed_frames": 22, "read_frames": 21, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.734, "pushed": 0.734, "read": 0.7}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.782, "count": 21, "max_ms": 15.623, "p95_ms": 14.325}, "gstreamer_push": {"avg_ms": 1.01, "count": 22, "max_ms": 1.219, "p95_ms": 1.116}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.3, "count": 21, "max_ms": 74.482, "p95_ms": 66.325}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.191, "count": 21, "max_ms": 81.277, "p95_ms": 79.516}, "yolo11_640_fire_11class": {"avg_ms": 53.849, "count": 21, "max_ms": 73.761, "p95_ms": 63.665}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.242, "count": 21, "max_ms": 84.097, "p95_ms": 79.023}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.925, "count": 22, "max_ms": 171.635, "p95_ms": 169.062}, "read_interval": {"avg_ms": 1427.691, "count": 21, "max_ms": 16269.687, "p95_ms": 10622.988}, "triton_wait": {"avg_ms": 62.544, "count": 21, "max_ms": 84.109, "p95_ms": 79.528}, "visualize": {"avg_ms": 5.103, "count": 21, "max_ms": 9.172, "p95_ms": 6.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260703_090800_3920bd95.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 107, "tracking_summary": 20843}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 16527.895, "limit_mb": 8796093022207.996}, "detail": {"pid": 155316, "restart_count": 152, "start_count": 153}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:06:25+09:00", "last_error_summary": "[h264 @ 0x565391648ac0] error while decoding MB 11 37, bytestream -9 [in#0/rtsp @ 0x5653914fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653914fbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:06:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:06:27+09:00", "pid": 155316, "process": {"exists": true, "fd_count": 5, "hwm_mb": 106.762, "name": "ffmpeg", "pid": 155316, "rss_mb": 106.762, "state": "S (sleeping)", "threads": 19}, "restart_count": 152, "restart_count_per_hour": 4, "rtsp_timeout_count": 258, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 153}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.51, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2501274, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:06:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "3920bd95", "last_frame_pushed_age_sec": 4.06, "last_frame_pushed_at": "2026-07-06T09:06:25+09:00", "last_frame_read_age_sec": 4.211, "last_frame_read_at": "2026-07-06T09:06:25+09:00", "mjpeg": {"age_sec": 4.041, "exists": true, "mtime": "2026-07-06T09:06:25.427238+09:00", "mtime_epoch": 1783296385.4272375, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231942}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:06:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2572.191, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.624, "pushed": 0.624, "read": 0.624}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.927, "count": 14, "max_ms": 15.623, "p95_ms": 15.623}, "gstreamer_push": {"avg_ms": 1.026, "count": 14, "max_ms": 1.219, "p95_ms": 1.219}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.444, "count": 14, "max_ms": 74.482, "p95_ms": 74.482}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.225, "count": 14, "max_ms": 81.277, "p95_ms": 81.277}, "yolo11_640_fire_11class": {"avg_ms": 53.839, "count": 14, "max_ms": 73.761, "p95_ms": 73.761}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.345, "count": 14, "max_ms": 84.097, "p95_ms": 84.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.569, "count": 14, "max_ms": 171.635, "p95_ms": 171.635}, "read_interval": {"avg_ms": 2059.848, "count": 14, "max_ms": 16269.687, "p95_ms": 16269.687}, "triton_wait": {"avg_ms": 62.249, "count": 14, "max_ms": 84.109, "p95_ms": 84.109}, "visualize": {"avg_ms": 5.207, "count": 14, "max_ms": 9.172, "p95_ms": 9.172}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.426, "window_s": 30.0}, "started_at": "2026-07-03T09:08:00+09:00", "state": "running", "task_id": "12211685-c628-4e51-9913-70e75ac1816d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260703_090802_c19c5747.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 397, "tracking_summary": 16716}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 15478.918, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5567387a09c0] error while decoding MB 44 23, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 836, "h264_decode_error_count": 2043, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:12:58+09:00", "last_error_summary": "[h264 @ 0x5567387a09c0] error while decoding MB 44 23, bytestream -14", "last_exit_at": "2026-07-06T09:04:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 480.228, "last_started_at": "2026-07-06T09:04:57+09:00", "pid": 155209, "process": {"exists": false, "pid": 155209}, "restart_count": 929, "restart_count_per_hour": 7, "rtsp_timeout_count": 520, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 930}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.333, "sample_count": 2720, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2006005, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:12:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c19c5747", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T09:12:57+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-06T09:12:57+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T09:12:57.998142+09:00", "mtime_epoch": 1783296777.9981425, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192797}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T09:12:58+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 10019.148, "name": "celery", "pid": 237, "rss_mb": 9583.094, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 282, "pushed_frames": 282, "read_frames": 281, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.423, "pushed": 9.423, "read": 9.389}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.713, "count": 282, "max_ms": 35.395, "p95_ms": 29.434}, "gstreamer_push": {"avg_ms": 1.182, "count": 282, "max_ms": 1.381, "p95_ms": 1.28}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.191, "count": 282, "max_ms": 66.511, "p95_ms": 63.137}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.884, "count": 282, "max_ms": 80.502, "p95_ms": 70.34}, "yolo11_640_fire_11class": {"avg_ms": 50.943, "count": 282, "max_ms": 74.614, "p95_ms": 67.51}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.344, "count": 282, "max_ms": 78.258, "p95_ms": 74.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.698, "count": 282, "max_ms": 115.493, "p95_ms": 109.884}, "read_interval": {"avg_ms": 106.385, "count": 281, "max_ms": 129.876, "p95_ms": 124.689}, "triton_wait": {"avg_ms": 57.086, "count": 282, "max_ms": 80.515, "p95_ms": 75.385}, "visualize": {"avg_ms": 3.276, "count": 282, "max_ms": 4.235, "p95_ms": 3.545}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-03T09:08:02+09:00", "state": "running", "task_id": "9223e2cb-c5a3-47a3-ba90-7b570575e095", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_091257_fa72c4d8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 15674.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 155470, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:12:57+09:00", "pid": 155470, "process": {"exists": true, "fd_count": 5, "hwm_mb": 210.062, "name": "ffmpeg", "pid": 155470, "rss_mb": 210.062, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.939}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "fa72c4d8", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 2.99, "exists": true, "mtime": "2026-07-06T09:12:56.122109+09:00", "mtime_epoch": 1783296776.1221094, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 240529}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:12:59+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 1007.762, "name": "celery", "pid": 239, "rss_mb": 826.098, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T09:12:57+09:00", "state": "running", "task_id": "a1dbe723-60fb-4dca-a5de-f047e2679297", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_091258_2401d7bd.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 15788.227, "limit_mb": 8796093022207.996}, "detail": {"pid": 155623, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:12:59+09:00", "pid": 155623, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.82, "name": "ffmpeg", "pid": 155623, "rss_mb": 138.82, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.939}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "2401d7bd", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.003, "exists": true, "mtime": "2026-07-06T09:12:57.998142+09:00", "mtime_epoch": 1783296777.9981425, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192797}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T09:13:01+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 2838.816, "name": "celery", "pid": 267, "rss_mb": 2430.945, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T09:12:58+09:00", "state": "running", "task_id": "dbefc1c4-a354-4bb1-a826-b771712d28c3", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_092203_cc0f103b.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 2688.477, "limit_mb": 8796093022207.996}, "detail": {"pid": 275, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:22:03+09:00", "pid": 275, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.754, "name": "ffmpeg", "pid": 275, "rss_mb": 149.754, "state": "R (running)", "threads": 49}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.897}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "warning", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["mjpeg_age_sec>10s"]}, "instance_id": "cc0f103b", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 15.468, "exists": true, "mtime": "2026-07-06T09:21:50.109958+09:00", "mtime_epoch": 1783297310.1099584, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 235864}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:22:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 467.273, "name": "celery", "pid": 267, "rss_mb": 467.273, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_warning"], "recommended_action": "degrade_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T09:22:03+09:00", "state": "running", "task_id": "0848868f-dd6d-429c-840d-4fda5bb9bf97", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_092203_698cf579.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 2688.477, "limit_mb": 8796093022207.996}, "detail": {"pid": 276, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:22:03+09:00", "pid": 276, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.238, "name": "ffmpeg", "pid": 276, "rss_mb": 142.238, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.898}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "warning", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["mjpeg_age_sec>10s"]}, "instance_id": "698cf579", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 15.448, "exists": true, "mtime": "2026-07-06T09:21:50.129959+09:00", "mtime_epoch": 1783297310.1299589, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200663}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T09:22:05+09:00", "process": {"exists": true, "fd_count": 106, "hwm_mb": 467.172, "name": "celery", "pid": 237, "rss_mb": 467.172, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_warning"], "recommended_action": "degrade_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T09:22:03+09:00", "state": "running", "task_id": "01695c9d-6d29-4142-ad55-abcb8ed2c274", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3328.062, "limit_mb": 8796093022207.996}, "detail": {"pid": 779, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:33:49+09:00", "pid": 779, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.633, "name": "ffmpeg", "pid": 779, "rss_mb": 57.633, "state": "S (sleeping)", "threads": 1}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.928}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.018, "exists": true, "mtime": "2026-07-06T09:33:48.181934+09:00", "mtime_epoch": 1783298028.1819339, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224178}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:33:51+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 876.078, "name": "celery", "pid": 267, "rss_mb": 807.426, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 21}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3727.145, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a2efe55880] error while decoding MB 44 41, bytestream -5 [in#0/rtsp @ 0x55a2efd47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a2efd47e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:38:56+09:00", "last_error_summary": "[h264 @ 0x55a2efe55880] error while decoding MB 44 41, bytestream -5 [in#0/rtsp @ 0x55a2efd47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a2efd47e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 307.392, "last_started_at": "2026-07-06T09:33:49+09:00", "pid": 779, "process": {"exists": false, "pid": 779}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.74, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2608, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:38:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-06T09:38:56+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T09:38:56+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T09:38:56.532103+09:00", "mtime_epoch": 1783298336.5321033, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:38:56+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.586, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.692, "pushed": 0.692, "read": 0.75}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.255, "count": 12, "max_ms": 3.792, "p95_ms": 3.792}, "gstreamer_push": {"avg_ms": 1.324, "count": 12, "max_ms": 1.447, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.58, "count": 12, "max_ms": 61.515, "p95_ms": 61.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.858, "count": 12, "max_ms": 73.771, "p95_ms": 73.771}, "yolo11_640_fire_11class": {"avg_ms": 48.841, "count": 12, "max_ms": 60.948, "p95_ms": 60.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.985, "count": 12, "max_ms": 62.972, "p95_ms": 62.972}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.014, "count": 12, "max_ms": 85.754, "p95_ms": 85.754}, "read_interval": {"avg_ms": 2710.682, "count": 13, "max_ms": 17973.597, "p95_ms": 17973.597}, "triton_wait": {"avg_ms": 56.04, "count": 12, "max_ms": 73.785, "p95_ms": 73.785}, "visualize": {"avg_ms": 3.738, "count": 12, "max_ms": 4.318, "p95_ms": 4.318}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.33, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 21}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3726.906, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T09:38:56+09:00", "last_error_summary": "[h264 @ 0x55a2efe55880] error while decoding MB 44 41, bytestream -5 [in#0/rtsp @ 0x55a2efd47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a2efd47e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:38:56+09:00", "last_error_summary": "[h264 @ 0x55a2efe55880] error while decoding MB 44 41, bytestream -5 [in#0/rtsp @ 0x55a2efd47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a2efd47e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:38:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 307.418, "last_started_at": "2026-07-06T09:33:49+09:00", "pid": 779, "process": {"exists": false, "pid": 779}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.74, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2609, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:38:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T09:38:56+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-06T09:38:56+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-06T09:38:56.532103+09:00", "mtime_epoch": 1783298336.5321033, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:38:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.625, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.299, "count": 13, "max_ms": 3.837, "p95_ms": 3.837}, "gstreamer_push": {"avg_ms": 1.325, "count": 13, "max_ms": 1.447, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.564, "count": 13, "max_ms": 61.515, "p95_ms": 61.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.821, "count": 13, "max_ms": 73.771, "p95_ms": 73.771}, "yolo11_640_fire_11class": {"avg_ms": 48.925, "count": 13, "max_ms": 60.948, "p95_ms": 60.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.408, "count": 13, "max_ms": 67.473, "p95_ms": 67.473}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.099, "count": 13, "max_ms": 85.754, "p95_ms": 85.754}, "read_interval": {"avg_ms": 2710.682, "count": 13, "max_ms": 17973.597, "p95_ms": 17973.597}, "triton_wait": {"avg_ms": 56.92, "count": 13, "max_ms": 73.785, "p95_ms": 73.785}, "visualize": {"avg_ms": 3.899, "count": 13, "max_ms": 5.83, "p95_ms": 5.83}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.355, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 21}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3726.805, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:38:56+09:00", "last_error_summary": "[h264 @ 0x55a2efe55880] error while decoding MB 44 41, bytestream -5 [in#0/rtsp @ 0x55a2efd47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a2efd47e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:38:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 307.425, "last_started_at": "2026-07-06T09:33:49+09:00", "pid": 779, "process": {"exists": false, "pid": 779}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.74, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2609, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:38:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T09:38:56+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-06T09:38:56+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-06T09:38:56.532103+09:00", "mtime_epoch": 1783298336.5321033, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:38:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.625, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.299, "count": 13, "max_ms": 3.837, "p95_ms": 3.837}, "gstreamer_push": {"avg_ms": 1.325, "count": 13, "max_ms": 1.447, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.564, "count": 13, "max_ms": 61.515, "p95_ms": 61.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.821, "count": 13, "max_ms": 73.771, "p95_ms": 73.771}, "yolo11_640_fire_11class": {"avg_ms": 48.925, "count": 13, "max_ms": 60.948, "p95_ms": 60.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.408, "count": 13, "max_ms": 67.473, "p95_ms": 67.473}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.099, "count": 13, "max_ms": 85.754, "p95_ms": 85.754}, "read_interval": {"avg_ms": 2710.682, "count": 13, "max_ms": 17973.597, "p95_ms": 17973.597}, "triton_wait": {"avg_ms": 56.92, "count": 13, "max_ms": 73.785, "p95_ms": 73.785}, "visualize": {"avg_ms": 3.899, "count": 13, "max_ms": 5.83, "p95_ms": 5.83}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.363, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 21}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3750.555, "limit_mb": 8796093022207.996}, "detail": {"pid": 993, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:38:56+09:00", "last_error_summary": "[h264 @ 0x55a2efe55880] error while decoding MB 44 41, bytestream -5 [in#0/rtsp @ 0x55a2efd47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a2efd47e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:38:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:38:59+09:00", "pid": 993, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.586, "name": "ffmpeg", "pid": 993, "rss_mb": 58.586, "state": "S (sleeping)", "threads": 1}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.786, "sample_count": 2512, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2609, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:38:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 4.602, "last_frame_pushed_at": "2026-07-06T09:38:56+09:00", "last_frame_read_age_sec": 4.686, "last_frame_read_at": "2026-07-06T09:38:56+09:00", "mjpeg": {"age_sec": 4.585, "exists": true, "mtime": "2026-07-06T09:38:56.628105+09:00", "mtime_epoch": 1783298336.6281052, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230952}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:39:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.625, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.592, "pushed": 0.592, "read": 0.592}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.299, "count": 13, "max_ms": 3.837, "p95_ms": 3.837}, "gstreamer_push": {"avg_ms": 1.325, "count": 13, "max_ms": 1.447, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.564, "count": 13, "max_ms": 61.515, "p95_ms": 61.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.821, "count": 13, "max_ms": 73.771, "p95_ms": 73.771}, "yolo11_640_fire_11class": {"avg_ms": 48.925, "count": 13, "max_ms": 60.948, "p95_ms": 60.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.408, "count": 13, "max_ms": 67.473, "p95_ms": 67.473}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.099, "count": 13, "max_ms": 85.754, "p95_ms": 85.754}, "read_interval": {"avg_ms": 2710.682, "count": 13, "max_ms": 17973.597, "p95_ms": 17973.597}, "triton_wait": {"avg_ms": 56.92, "count": 13, "max_ms": 73.785, "p95_ms": 73.785}, "visualize": {"avg_ms": 3.899, "count": 13, "max_ms": 5.83, "p95_ms": 5.83}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.951, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260706_094104_726fba62.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4152.762, "limit_mb": 8796093022207.996}, "detail": {"pid": 1150, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:41:04+09:00", "pid": 1150, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.723, "name": "ffmpeg", "pid": 1150, "rss_mb": 137.723, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.746}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "726fba62", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 260653.483, "exists": true, "mtime": "2026-07-03T09:16:52.608144+09:00", "mtime_epoch": 1783037812.6081438, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 262975}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-06T09:41:06+09:00", "process": {"exists": true, "fd_count": 110, "hwm_mb": 470.496, "name": "celery", "pid": 239, "rss_mb": 470.496, "state": "R (running)", "threads": 7}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T09:41:03+09:00", "state": "running", "task_id": "22600d6e-3b2b-439c-be8f-37c755ee4afb", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4196.59, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562196ca9e00] error while decoding MB 5 25, bytestream -15 [in#0/rtsp @ 0x5621963a1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5621963a1e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:42:31+09:00", "last_error_summary": "[h264 @ 0x562196ca9e00] error while decoding MB 5 25, bytestream -15 [in#0/rtsp @ 0x5621963a1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5621963a1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:38:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 212.053, "last_started_at": "2026-07-06T09:38:59+09:00", "pid": 993, "process": {"exists": false, "pid": 993}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.139, "sample_count": 2084, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4278, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:42:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.083, "last_frame_pushed_at": "2026-07-06T09:42:31+09:00", "last_frame_read_age_sec": 0.071, "last_frame_read_at": "2026-07-06T09:42:31+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T09:42:31.208297+09:00", "mtime_epoch": 1783298551.2082968, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228750}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:42:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.625, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.732, "pushed": 0.732, "read": 0.788}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.081, "count": 13, "max_ms": 8.825, "p95_ms": 8.825}, "gstreamer_push": {"avg_ms": 1.266, "count": 13, "max_ms": 1.522, "p95_ms": 1.522}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.905, "count": 13, "max_ms": 71.078, "p95_ms": 71.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.852, "count": 13, "max_ms": 90.982, "p95_ms": 90.982}, "yolo11_640_fire_11class": {"avg_ms": 55.377, "count": 13, "max_ms": 71.066, "p95_ms": 71.066}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.335, "count": 13, "max_ms": 77.639, "p95_ms": 77.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.561, "count": 13, "max_ms": 111.132, "p95_ms": 111.132}, "read_interval": {"avg_ms": 2416.262, "count": 14, "max_ms": 16492.469, "p95_ms": 16492.469}, "triton_wait": {"avg_ms": 69.933, "count": 13, "max_ms": 90.995, "p95_ms": 90.995}, "visualize": {"avg_ms": 4.151, "count": 13, "max_ms": 6.995, "p95_ms": 6.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.765, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4196.086, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T09:42:31+09:00", "last_error_summary": "[h264 @ 0x562196ca9e00] error while decoding MB 5 25, bytestream -15 [in#0/rtsp @ 0x5621963a1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5621963a1e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:42:31+09:00", "last_error_summary": "[h264 @ 0x562196ca9e00] error while decoding MB 5 25, bytestream -15 [in#0/rtsp @ 0x5621963a1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5621963a1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:42:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 212.083, "last_started_at": "2026-07-06T09:38:59+09:00", "pid": 993, "process": {"exists": false, "pid": 993}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.139, "sample_count": 2085, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4279, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:42:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T09:42:31+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-06T09:42:31+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-06T09:42:31.208297+09:00", "mtime_epoch": 1783298551.2082968, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228750}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:42:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.787, "pushed": 0.787, "read": 0.787}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.02, "count": 14, "max_ms": 8.825, "p95_ms": 8.825}, "gstreamer_push": {"avg_ms": 1.279, "count": 14, "max_ms": 1.522, "p95_ms": 1.522}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.464, "count": 14, "max_ms": 71.078, "p95_ms": 71.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.66, "count": 14, "max_ms": 90.982, "p95_ms": 90.982}, "yolo11_640_fire_11class": {"avg_ms": 56.073, "count": 14, "max_ms": 71.066, "p95_ms": 71.066}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.62, "count": 14, "max_ms": 77.639, "p95_ms": 77.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.656, "count": 14, "max_ms": 111.132, "p95_ms": 111.132}, "read_interval": {"avg_ms": 2416.262, "count": 14, "max_ms": 16492.469, "p95_ms": 16492.469}, "triton_wait": {"avg_ms": 70.736, "count": 14, "max_ms": 90.995, "p95_ms": 90.995}, "visualize": {"avg_ms": 4.112, "count": 14, "max_ms": 6.995, "p95_ms": 6.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.793, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4195.844, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:42:31+09:00", "last_error_summary": "[h264 @ 0x562196ca9e00] error while decoding MB 5 25, bytestream -15 [in#0/rtsp @ 0x5621963a1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5621963a1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:42:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 212.089, "last_started_at": "2026-07-06T09:38:59+09:00", "pid": 993, "process": {"exists": false, "pid": 993}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.139, "sample_count": 2085, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4279, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:42:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T09:42:31+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-06T09:42:31+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-06T09:42:31.208297+09:00", "mtime_epoch": 1783298551.2082968, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228750}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:42:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.787, "pushed": 0.787, "read": 0.787}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.02, "count": 14, "max_ms": 8.825, "p95_ms": 8.825}, "gstreamer_push": {"avg_ms": 1.279, "count": 14, "max_ms": 1.522, "p95_ms": 1.522}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.464, "count": 14, "max_ms": 71.078, "p95_ms": 71.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.66, "count": 14, "max_ms": 90.982, "p95_ms": 90.982}, "yolo11_640_fire_11class": {"avg_ms": 56.073, "count": 14, "max_ms": 71.066, "p95_ms": 71.066}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.62, "count": 14, "max_ms": 77.639, "p95_ms": 77.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.656, "count": 14, "max_ms": 111.132, "p95_ms": 111.132}, "read_interval": {"avg_ms": 2416.262, "count": 14, "max_ms": 16492.469, "p95_ms": 16492.469}, "triton_wait": {"avg_ms": 70.736, "count": 14, "max_ms": 90.995, "p95_ms": 90.995}, "visualize": {"avg_ms": 4.112, "count": 14, "max_ms": 6.995, "p95_ms": 6.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.8, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4364.016, "limit_mb": 8796093022207.996}, "detail": {"pid": 1417, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:42:31+09:00", "last_error_summary": "[h264 @ 0x562196ca9e00] error while decoding MB 5 25, bytestream -15 [in#0/rtsp @ 0x5621963a1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5621963a1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:42:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:42:34+09:00", "pid": 1417, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.793, "name": "ffmpeg", "pid": 1417, "rss_mb": 197.793, "state": "S (sleeping)", "threads": 106}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.43, "sample_count": 2034, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4279, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:42:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 4.809, "last_frame_pushed_at": "2026-07-06T09:42:31+09:00", "last_frame_read_age_sec": 4.904, "last_frame_read_at": "2026-07-06T09:42:31+09:00", "mjpeg": {"age_sec": 4.786, "exists": true, "mtime": "2026-07-06T09:42:31.312299+09:00", "mtime_epoch": 1783298551.3122988, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224518}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:42:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.62, "pushed": 0.62, "read": 0.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.02, "count": 14, "max_ms": 8.825, "p95_ms": 8.825}, "gstreamer_push": {"avg_ms": 1.279, "count": 14, "max_ms": 1.522, "p95_ms": 1.522}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.464, "count": 14, "max_ms": 71.078, "p95_ms": 71.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.66, "count": 14, "max_ms": 90.982, "p95_ms": 90.982}, "yolo11_640_fire_11class": {"avg_ms": 56.073, "count": 14, "max_ms": 71.066, "p95_ms": 71.066}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.62, "count": 14, "max_ms": 77.639, "p95_ms": 77.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.656, "count": 14, "max_ms": 111.132, "p95_ms": 111.132}, "read_interval": {"avg_ms": 2416.262, "count": 14, "max_ms": 16492.469, "p95_ms": 16492.469}, "triton_wait": {"avg_ms": 70.736, "count": 14, "max_ms": 90.995, "p95_ms": 90.995}, "visualize": {"avg_ms": 4.112, "count": 14, "max_ms": 6.995, "p95_ms": 6.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.597, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4245.516, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x5576a86b2200] missing picture in access unit with size 18 [h264 @ 0x5576a87dbd00] no frame! [in#0/rtsp @ 0x5576a86ade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576a86ade00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5576a86b2840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:43:57+09:00", "last_error_summary": "[NULL @ 0x5576a86b2200] missing picture in access unit with size 18 [h264 @ 0x5576a87dbd00] no frame! [in#0/rtsp @ 0x5576a86ade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576a86ade00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5576a86b2840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T09:42:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 83.694, "last_started_at": "2026-07-06T09:42:34+09:00", "pid": 1417, "process": {"exists": false, "pid": 1417}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.896, "sample_count": 2060, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4669, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-06T09:43:57+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T09:43:57+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-06T09:43:57.737972+09:00", "mtime_epoch": 1783298637.7379715, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239639}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:43:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.464, "pushed": 0.464, "read": 0.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.675, "count": 13, "max_ms": 8.031, "p95_ms": 8.031}, "gstreamer_push": {"avg_ms": 1.166, "count": 13, "max_ms": 1.288, "p95_ms": 1.288}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.642, "count": 13, "max_ms": 72.891, "p95_ms": 72.891}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.644, "count": 13, "max_ms": 91.863, "p95_ms": 91.863}, "yolo11_640_fire_11class": {"avg_ms": 55.772, "count": 13, "max_ms": 72.174, "p95_ms": 72.174}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.493, "count": 13, "max_ms": 82.144, "p95_ms": 82.144}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.535, "count": 13, "max_ms": 169.085, "p95_ms": 169.085}, "read_interval": {"avg_ms": 2370.038, "count": 14, "max_ms": 16194.45, "p95_ms": 16194.45}, "triton_wait": {"avg_ms": 77.657, "count": 13, "max_ms": 91.876, "p95_ms": 91.876}, "visualize": {"avg_ms": 5.33, "count": 13, "max_ms": 6.542, "p95_ms": 6.542}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.026, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4244.566, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T09:43:57+09:00", "last_error_summary": "[NULL @ 0x5576a86b2200] missing picture in access unit with size 18 [h264 @ 0x5576a87dbd00] no frame! [in#0/rtsp @ 0x5576a86ade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576a86ade00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5576a86b2840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:43:57+09:00", "last_error_summary": "[NULL @ 0x5576a86b2200] missing picture in access unit with size 18 [h264 @ 0x5576a87dbd00] no frame! [in#0/rtsp @ 0x5576a86ade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576a86ade00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5576a86b2840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T09:43:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 83.801, "last_started_at": "2026-07-06T09:42:34+09:00", "pid": 1417, "process": {"exists": false, "pid": 1417}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.958, "sample_count": 2061, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4670, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T09:43:57+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-06T09:43:57+09:00", "mjpeg": {"age_sec": 0.16, "exists": true, "mtime": "2026-07-06T09:43:57.737972+09:00", "mtime_epoch": 1783298637.7379715, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239639}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:43:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.637, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.498, "pushed": 0.498, "read": 0.498}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.739, "count": 14, "max_ms": 8.031, "p95_ms": 8.031}, "gstreamer_push": {"avg_ms": 1.158, "count": 14, "max_ms": 1.288, "p95_ms": 1.288}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.144, "count": 14, "max_ms": 72.891, "p95_ms": 72.891}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.656, "count": 14, "max_ms": 91.863, "p95_ms": 91.863}, "yolo11_640_fire_11class": {"avg_ms": 55.202, "count": 14, "max_ms": 72.174, "p95_ms": 72.174}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.356, "count": 14, "max_ms": 82.144, "p95_ms": 82.144}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.413, "count": 14, "max_ms": 169.085, "p95_ms": 169.085}, "read_interval": {"avg_ms": 2370.038, "count": 14, "max_ms": 16194.45, "p95_ms": 16194.45}, "triton_wait": {"avg_ms": 77.668, "count": 14, "max_ms": 91.876, "p95_ms": 91.876}, "visualize": {"avg_ms": 5.314, "count": 14, "max_ms": 6.542, "p95_ms": 6.542}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.132, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4244.559, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:43:57+09:00", "last_error_summary": "[NULL @ 0x5576a86b2200] missing picture in access unit with size 18 [h264 @ 0x5576a87dbd00] no frame! [in#0/rtsp @ 0x5576a86ade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576a86ade00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5576a86b2840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T09:43:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 83.804, "last_started_at": "2026-07-06T09:42:34+09:00", "pid": 1417, "process": {"exists": false, "pid": 1417}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.958, "sample_count": 2061, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4670, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T09:43:57+09:00", "last_frame_read_age_sec": 0.172, "last_frame_read_at": "2026-07-06T09:43:57+09:00", "mjpeg": {"age_sec": 0.163, "exists": true, "mtime": "2026-07-06T09:43:57.737972+09:00", "mtime_epoch": 1783298637.7379715, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239639}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:43:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.637, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.498, "pushed": 0.498, "read": 0.498}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.739, "count": 14, "max_ms": 8.031, "p95_ms": 8.031}, "gstreamer_push": {"avg_ms": 1.158, "count": 14, "max_ms": 1.288, "p95_ms": 1.288}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.144, "count": 14, "max_ms": 72.891, "p95_ms": 72.891}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.656, "count": 14, "max_ms": 91.863, "p95_ms": 91.863}, "yolo11_640_fire_11class": {"avg_ms": 55.202, "count": 14, "max_ms": 72.174, "p95_ms": 72.174}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.356, "count": 14, "max_ms": 82.144, "p95_ms": 82.144}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.413, "count": 14, "max_ms": 169.085, "p95_ms": 169.085}, "read_interval": {"avg_ms": 2370.038, "count": 14, "max_ms": 16194.45, "p95_ms": 16194.45}, "triton_wait": {"avg_ms": 77.668, "count": 14, "max_ms": 91.876, "p95_ms": 91.876}, "visualize": {"avg_ms": 5.314, "count": 14, "max_ms": 6.542, "p95_ms": 6.542}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.136, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4415.434, "limit_mb": 8796093022207.996}, "detail": {"pid": 1569, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:43:57+09:00", "last_error_summary": "[NULL @ 0x5576a86b2200] missing picture in access unit with size 18 [h264 @ 0x5576a87dbd00] no frame! [in#0/rtsp @ 0x5576a86ade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576a86ade00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5576a86b2840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T09:44:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:44:00+09:00", "pid": 1569, "process": {"exists": true, "fd_count": 5, "hwm_mb": 201.727, "name": "ffmpeg", "pid": 1569, "rss_mb": 201.727, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.958, "sample_count": 2052, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4670, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 4.896, "last_frame_pushed_at": "2026-07-06T09:43:57+09:00", "last_frame_read_age_sec": 5.062, "last_frame_read_at": "2026-07-06T09:43:57+09:00", "mjpeg": {"age_sec": 4.881, "exists": true, "mtime": "2026-07-06T09:43:57.909975+09:00", "mtime_epoch": 1783298637.9099748, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239457}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:44:02+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1215.637, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.568, "pushed": 0.568, "read": 0.568}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.763, "count": 13, "max_ms": 8.031, "p95_ms": 8.031}, "gstreamer_push": {"avg_ms": 1.161, "count": 13, "max_ms": 1.288, "p95_ms": 1.288}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.575, "count": 13, "max_ms": 72.891, "p95_ms": 72.891}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.211, "count": 13, "max_ms": 91.863, "p95_ms": 91.863}, "yolo11_640_fire_11class": {"avg_ms": 55.675, "count": 13, "max_ms": 72.174, "p95_ms": 72.174}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.843, "count": 13, "max_ms": 82.144, "p95_ms": 82.144}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.179, "count": 13, "max_ms": 169.085, "p95_ms": 169.085}, "read_interval": {"avg_ms": 2151.048, "count": 13, "max_ms": 16194.45, "p95_ms": 16194.45}, "triton_wait": {"avg_ms": 79.224, "count": 13, "max_ms": 91.876, "p95_ms": 91.876}, "visualize": {"avg_ms": 5.342, "count": 13, "max_ms": 6.542, "p95_ms": 6.542}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.907, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 67}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4348.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fec0aae840] error while decoding MB 6 66, bytestream -15 [in#0/rtsp @ 0x55fec0994e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55fec0994e00] Error retrieving a packet from demuxer: Input/output error", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:50:34+09:00", "last_error_summary": "[h264 @ 0x55fec0aae840] error while decoding MB 6 66, bytestream -15 [in#0/rtsp @ 0x55fec0994e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55fec0994e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-06T09:44:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 393.721, "last_started_at": "2026-07-06T09:44:00+09:00", "pid": 1569, "process": {"exists": false, "pid": 1569}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.635, "sample_count": 2552, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8119, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:50:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-06T09:50:34+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T09:50:34+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-06T09:50:34.461575+09:00", "mtime_epoch": 1783299034.4615753, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 237150}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:50:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1217.477, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 56, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.87, "pushed": 1.87, "read": 1.87}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.736, "count": 56, "max_ms": 5.579, "p95_ms": 5.436}, "gstreamer_push": {"avg_ms": 1.329, "count": 56, "max_ms": 1.501, "p95_ms": 1.475}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.373, "count": 56, "max_ms": 73.399, "p95_ms": 65.049}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.645, "count": 56, "max_ms": 109.689, "p95_ms": 99.8}, "yolo11_640_fire_11class": {"avg_ms": 52.674, "count": 56, "max_ms": 72.535, "p95_ms": 66.065}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.527, "count": 56, "max_ms": 75.567, "p95_ms": 67.145}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.136, "count": 56, "max_ms": 142.046, "p95_ms": 133.642}, "read_interval": {"avg_ms": 535.436, "count": 56, "max_ms": 15486.66, "p95_ms": 147.239}, "triton_wait": {"avg_ms": 68.658, "count": 56, "max_ms": 109.702, "p95_ms": 99.813}, "visualize": {"avg_ms": 3.983, "count": 56, "max_ms": 5.754, "p95_ms": 5.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 67}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4346.938, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T09:50:34+09:00", "last_error_summary": "[h264 @ 0x55fec0aae840] error while decoding MB 6 66, bytestream -15 [in#0/rtsp @ 0x55fec0994e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55fec0994e00] Error retrieving a packet from demuxer: Input/output error", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:50:34+09:00", "last_error_summary": "[h264 @ 0x55fec0aae840] error while decoding MB 6 66, bytestream -15 [in#0/rtsp @ 0x55fec0994e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55fec0994e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-06T09:50:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 393.794, "last_started_at": "2026-07-06T09:44:00+09:00", "pid": 1569, "process": {"exists": false, "pid": 1569}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.635, "sample_count": 2552, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8120, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:50:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T09:50:34+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T09:50:34+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-06T09:50:34.461575+09:00", "mtime_epoch": 1783299034.4615753, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 237150}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:50:34+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1217.477, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.872, "pushed": 1.872, "read": 1.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.772, "count": 56, "max_ms": 5.579, "p95_ms": 5.436}, "gstreamer_push": {"avg_ms": 1.327, "count": 56, "max_ms": 1.501, "p95_ms": 1.475}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.125, "count": 56, "max_ms": 73.399, "p95_ms": 65.049}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.562, "count": 56, "max_ms": 109.689, "p95_ms": 99.8}, "yolo11_640_fire_11class": {"avg_ms": 52.491, "count": 56, "max_ms": 72.535, "p95_ms": 66.065}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.272, "count": 56, "max_ms": 75.567, "p95_ms": 67.145}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.719, "count": 56, "max_ms": 142.046, "p95_ms": 133.642}, "read_interval": {"avg_ms": 543.183, "count": 55, "max_ms": 15486.66, "p95_ms": 147.239}, "triton_wait": {"avg_ms": 68.575, "count": 56, "max_ms": 109.702, "p95_ms": 99.813}, "visualize": {"avg_ms": 4.008, "count": 56, "max_ms": 5.754, "p95_ms": 5.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 67}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4346.859, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:50:34+09:00", "last_error_summary": "[h264 @ 0x55fec0aae840] error while decoding MB 6 66, bytestream -15 [in#0/rtsp @ 0x55fec0994e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55fec0994e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-06T09:50:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 393.8, "last_started_at": "2026-07-06T09:44:00+09:00", "pid": 1569, "process": {"exists": false, "pid": 1569}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.635, "sample_count": 2552, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8120, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:50:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T09:50:34+09:00", "last_frame_read_age_sec": 0.135, "last_frame_read_at": "2026-07-06T09:50:34+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-06T09:50:34.461575+09:00", "mtime_epoch": 1783299034.4615753, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 237150}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:50:34+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1217.477, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.871, "pushed": 1.871, "read": 1.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.772, "count": 56, "max_ms": 5.579, "p95_ms": 5.436}, "gstreamer_push": {"avg_ms": 1.327, "count": 56, "max_ms": 1.501, "p95_ms": 1.475}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.125, "count": 56, "max_ms": 73.399, "p95_ms": 65.049}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.562, "count": 56, "max_ms": 109.689, "p95_ms": 99.8}, "yolo11_640_fire_11class": {"avg_ms": 52.491, "count": 56, "max_ms": 72.535, "p95_ms": 66.065}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.272, "count": 56, "max_ms": 75.567, "p95_ms": 67.145}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.719, "count": 56, "max_ms": 142.046, "p95_ms": 133.642}, "read_interval": {"avg_ms": 543.183, "count": 55, "max_ms": 15486.66, "p95_ms": 147.239}, "triton_wait": {"avg_ms": 68.575, "count": 56, "max_ms": 109.702, "p95_ms": 99.813}, "visualize": {"avg_ms": 4.008, "count": 56, "max_ms": 5.754, "p95_ms": 5.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 67}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4369.891, "limit_mb": 8796093022207.996}, "detail": {"pid": 1721, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:50:34+09:00", "last_error_summary": "[h264 @ 0x55fec0aae840] error while decoding MB 6 66, bytestream -15 [in#0/rtsp @ 0x55fec0994e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55fec0994e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-06T09:50:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:50:37+09:00", "pid": 1721, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.328, "name": "ffmpeg", "pid": 1721, "rss_mb": 58.328, "state": "S (sleeping)", "threads": 1}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.809, "sample_count": 2503, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8120, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:50:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 4.666, "last_frame_pushed_at": "2026-07-06T09:50:34+09:00", "last_frame_read_age_sec": 4.791, "last_frame_read_at": "2026-07-06T09:50:34+09:00", "mjpeg": {"age_sec": 4.648, "exists": true, "mtime": "2026-07-06T09:50:34.597578+09:00", "mtime_epoch": 1783299034.5975778, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:50:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1217.477, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.65, "pushed": 0.65, "read": 0.65}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.938, "count": 14, "max_ms": 5.579, "p95_ms": 5.579}, "gstreamer_push": {"avg_ms": 1.297, "count": 14, "max_ms": 1.462, "p95_ms": 1.462}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.985, "count": 14, "max_ms": 73.399, "p95_ms": 73.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.805, "count": 14, "max_ms": 109.689, "p95_ms": 109.689}, "yolo11_640_fire_11class": {"avg_ms": 53.604, "count": 14, "max_ms": 72.535, "p95_ms": 72.535}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.458, "count": 14, "max_ms": 75.567, "p95_ms": 75.567}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.315, "count": 14, "max_ms": 122.42, "p95_ms": 122.42}, "read_interval": {"avg_ms": 1834.3, "count": 14, "max_ms": 15486.66, "p95_ms": 15486.66}, "triton_wait": {"avg_ms": 69.819, "count": 14, "max_ms": 109.702, "p95_ms": 109.702}, "visualize": {"avg_ms": 4.082, "count": 14, "max_ms": 5.256, "p95_ms": 5.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.53, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 79}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4350.621, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55610dd85680] error while decoding MB 34 53, bytestream -17 [in#0/rtsp @ 0x55610dbf2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55610dbf2e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:53:51+09:00", "last_error_summary": "[h264 @ 0x55610dd85680] error while decoding MB 34 53, bytestream -17 [in#0/rtsp @ 0x55610dbf2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55610dbf2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:50:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.784, "last_started_at": "2026-07-06T09:50:37+09:00", "pid": 1721, "process": {"exists": false, "pid": 1721}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.769, "sample_count": 2088, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9508, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:53:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T09:53:50+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T09:53:50+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T09:53:50.978277+09:00", "mtime_epoch": 1783299230.9782774, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229248}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:53:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1218.25, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 17, "pushed_frames": 17, "read_frames": 17, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.569, "pushed": 0.569, "read": 0.569}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.388, "count": 17, "max_ms": 6.888, "p95_ms": 6.888}, "gstreamer_push": {"avg_ms": 1.19, "count": 17, "max_ms": 1.59, "p95_ms": 1.59}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.593, "count": 17, "max_ms": 65.368, "p95_ms": 65.368}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.939, "count": 17, "max_ms": 103.166, "p95_ms": 103.166}, "yolo11_640_fire_11class": {"avg_ms": 48.983, "count": 17, "max_ms": 66.718, "p95_ms": 66.718}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.651, "count": 17, "max_ms": 64.614, "p95_ms": 64.614}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.483, "count": 17, "max_ms": 155.193, "p95_ms": 155.193}, "read_interval": {"avg_ms": 1761.613, "count": 17, "max_ms": 17848.213, "p95_ms": 17848.213}, "triton_wait": {"avg_ms": 70.952, "count": 17, "max_ms": 103.179, "p95_ms": 103.179}, "visualize": {"avg_ms": 5.184, "count": 17, "max_ms": 7.328, "p95_ms": 7.328}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 79}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4349.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T09:53:51+09:00", "last_error_summary": "[h264 @ 0x55610dd85680] error while decoding MB 34 53, bytestream -17 [in#0/rtsp @ 0x55610dbf2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55610dbf2e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:53:51+09:00", "last_error_summary": "[h264 @ 0x55610dd85680] error while decoding MB 34 53, bytestream -17 [in#0/rtsp @ 0x55610dbf2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55610dbf2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:53:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.873, "last_started_at": "2026-07-06T09:50:37+09:00", "pid": 1721, "process": {"exists": false, "pid": 1721}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.769, "sample_count": 2088, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9509, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:53:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T09:53:51+09:00", "last_frame_read_age_sec": 0.15, "last_frame_read_at": "2026-07-06T09:53:50+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-06T09:53:50.978277+09:00", "mtime_epoch": 1783299230.9782774, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229248}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:53:51+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1218.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 18, "pushed_frames": 18, "read_frames": 17, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.6, "pushed": 0.6, "read": 0.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.305, "count": 17, "max_ms": 5.558, "p95_ms": 5.558}, "gstreamer_push": {"avg_ms": 1.188, "count": 18, "max_ms": 1.59, "p95_ms": 1.59}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.809, "count": 17, "max_ms": 65.368, "p95_ms": 65.368}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.224, "count": 17, "max_ms": 103.166, "p95_ms": 103.166}, "yolo11_640_fire_11class": {"avg_ms": 49.117, "count": 17, "max_ms": 66.718, "p95_ms": 66.718}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.475, "count": 17, "max_ms": 64.614, "p95_ms": 64.614}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.872, "count": 18, "max_ms": 155.193, "p95_ms": 155.193}, "read_interval": {"avg_ms": 1761.613, "count": 17, "max_ms": 17848.213, "p95_ms": 17848.213}, "triton_wait": {"avg_ms": 73.237, "count": 17, "max_ms": 103.179, "p95_ms": 103.179}, "visualize": {"avg_ms": 5.25, "count": 17, "max_ms": 7.328, "p95_ms": 7.328}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 79}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4349.707, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:53:51+09:00", "last_error_summary": "[h264 @ 0x55610dd85680] error while decoding MB 34 53, bytestream -17 [in#0/rtsp @ 0x55610dbf2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55610dbf2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:53:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.878, "last_started_at": "2026-07-06T09:50:37+09:00", "pid": 1721, "process": {"exists": false, "pid": 1721}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.769, "sample_count": 2088, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9509, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:53:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T09:53:51+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-06T09:53:50+09:00", "mjpeg": {"age_sec": 0.144, "exists": true, "mtime": "2026-07-06T09:53:50.978277+09:00", "mtime_epoch": 1783299230.9782774, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229248}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:53:51+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1218.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 18, "pushed_frames": 18, "read_frames": 17, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.6, "pushed": 0.6, "read": 0.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.305, "count": 17, "max_ms": 5.558, "p95_ms": 5.558}, "gstreamer_push": {"avg_ms": 1.188, "count": 18, "max_ms": 1.59, "p95_ms": 1.59}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.809, "count": 17, "max_ms": 65.368, "p95_ms": 65.368}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.224, "count": 17, "max_ms": 103.166, "p95_ms": 103.166}, "yolo11_640_fire_11class": {"avg_ms": 49.117, "count": 17, "max_ms": 66.718, "p95_ms": 66.718}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.475, "count": 17, "max_ms": 64.614, "p95_ms": 64.614}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.872, "count": 18, "max_ms": 155.193, "p95_ms": 155.193}, "read_interval": {"avg_ms": 1761.613, "count": 17, "max_ms": 17848.213, "p95_ms": 17848.213}, "triton_wait": {"avg_ms": 73.237, "count": 17, "max_ms": 103.179, "p95_ms": 103.179}, "visualize": {"avg_ms": 5.25, "count": 17, "max_ms": 7.328, "p95_ms": 7.328}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_093349_6a8c0861.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 79}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4371.094, "limit_mb": 8796093022207.996}, "detail": {"pid": 1873, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T09:53:51+09:00", "last_error_summary": "[h264 @ 0x55610dd85680] error while decoding MB 34 53, bytestream -17 [in#0/rtsp @ 0x55610dbf2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55610dbf2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T09:53:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:53:53+09:00", "pid": 1873, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.82, "name": "ffmpeg", "pid": 1873, "rss_mb": 57.82, "state": "S (sleeping)", "threads": 1}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.669, "sample_count": 2054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9509, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T09:53:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "6a8c0861", "last_frame_pushed_age_sec": 4.227, "last_frame_pushed_at": "2026-07-06T09:53:51+09:00", "last_frame_read_age_sec": 4.373, "last_frame_read_at": "2026-07-06T09:53:50+09:00", "mjpeg": {"age_sec": 4.207, "exists": true, "mtime": "2026-07-06T09:53:51.134277+09:00", "mtime_epoch": 1783299231.1342773, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228949}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T09:53:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1218.25, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.815, "pushed": 0.815, "read": 0.815}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.244, "count": 13, "max_ms": 5.555, "p95_ms": 5.555}, "gstreamer_push": {"avg_ms": 1.188, "count": 13, "max_ms": 1.59, "p95_ms": 1.59}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.537, "count": 13, "max_ms": 65.368, "p95_ms": 65.368}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.173, "count": 13, "max_ms": 103.166, "p95_ms": 103.166}, "yolo11_640_fire_11class": {"avg_ms": 49.569, "count": 13, "max_ms": 66.718, "p95_ms": 66.718}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.994, "count": 13, "max_ms": 64.614, "p95_ms": 64.614}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.029, "count": 13, "max_ms": 155.193, "p95_ms": 155.193}, "read_interval": {"avg_ms": 2263.022, "count": 13, "max_ms": 17848.213, "p95_ms": 17848.213}, "triton_wait": {"avg_ms": 73.186, "count": 13, "max_ms": 103.179, "p95_ms": 103.179}, "visualize": {"avg_ms": 5.291, "count": 13, "max_ms": 7.328, "p95_ms": 7.328}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.945, "window_s": 30.0}, "started_at": "2026-07-06T09:33:49+09:00", "state": "running", "task_id": "b9c05f6d-2f1c-4e8f-804b-13e47c6559ba", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_095424_2d4dcbc1.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4476.395, "limit_mb": 8796093022207.996}, "detail": {"pid": 2026, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T09:54:24+09:00", "pid": 2026, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.539, "name": "ffmpeg", "pid": 2026, "rss_mb": 141.539, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.927}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "2d4dcbc1", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.026, "exists": true, "mtime": "2026-07-06T09:54:23.110246+09:00", "mtime_epoch": 1783299263.110246, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188443}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T09:54:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 742.301, "name": "celery", "pid": 237, "rss_mb": 710.781, "state": "R (running)", "threads": 65}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T09:54:24+09:00", "state": "running", "task_id": "81ba486a-834a-4dad-9453-510df63a303b", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4642.793, "limit_mb": 8796093022207.996}, "detail": {"pid": 2190, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T10:04:24+09:00", "pid": 2190, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.602, "name": "ffmpeg", "pid": 2190, "rss_mb": 194.602, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.93}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c9313603", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.013, "exists": true, "mtime": "2026-07-06T10:04:23.834044+09:00", "mtime_epoch": 1783299863.8340442, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 225944}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:04:26+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1255.988, "name": "celery", "pid": 267, "rss_mb": 1221.707, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_100440_8fc5102c.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4744.707, "limit_mb": 8796093022207.996}, "detail": {"pid": 2344, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T10:04:40+09:00", "pid": 2344, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.164, "name": "ffmpeg", "pid": 2344, "rss_mb": 139.164, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.928}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "8fc5102c", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.024, "exists": true, "mtime": "2026-07-06T10:04:39.322215+09:00", "mtime_epoch": 1783299879.3222146, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182159}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T10:04:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.164, "name": "celery", "pid": 237, "rss_mb": 829.688, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T10:04:40+09:00", "state": "running", "task_id": "b7bb9bc3-2529-4776-bd26-c9fe21c510c6", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 14}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4612.07, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a6a5e6ac00] error while decoding MB 2 27, bytestream -13 [in#0/rtsp @ 0x55a6a5638e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6a5638e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:08:14+09:00", "last_error_summary": "[h264 @ 0x55a6a5e6ac00] error while decoding MB 2 27, bytestream -13 [in#0/rtsp @ 0x55a6a5638e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6a5638e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 229.861, "last_started_at": "2026-07-06T10:04:24+09:00", "pid": 2190, "process": {"exists": false, "pid": 2190}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.427, "sample_count": 1738, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 70.068}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1738, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:08:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-06T10:08:14+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T10:08:14+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-06T10:08:14.644794+09:00", "mtime_epoch": 1783300094.6447937, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211466}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:08:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1378.316, "name": "celery", "pid": 267, "rss_mb": 1369.652, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 62, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.073, "pushed": 2.073, "read": 2.073}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.159, "count": 62, "max_ms": 8.799, "p95_ms": 6.124}, "gstreamer_push": {"avg_ms": 1.411, "count": 62, "max_ms": 4.027, "p95_ms": 2.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.092, "count": 62, "max_ms": 70.279, "p95_ms": 65.065}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.663, "count": 62, "max_ms": 93.701, "p95_ms": 91.74}, "yolo11_640_fire_11class": {"avg_ms": 52.541, "count": 62, "max_ms": 69.229, "p95_ms": 64.305}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.281, "count": 62, "max_ms": 71.856, "p95_ms": 61.956}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.685, "count": 62, "max_ms": 146.975, "p95_ms": 136.709}, "read_interval": {"avg_ms": 483.026, "count": 62, "max_ms": 13275.316, "p95_ms": 157.168}, "triton_wait": {"avg_ms": 66.724, "count": 62, "max_ms": 93.714, "p95_ms": 91.757}, "visualize": {"avg_ms": 4.24, "count": 62, "max_ms": 6.583, "p95_ms": 5.637}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 14}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4611.246, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T10:08:14+09:00", "last_error_summary": "[h264 @ 0x55a6a5e6ac00] error while decoding MB 2 27, bytestream -13 [in#0/rtsp @ 0x55a6a5638e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6a5638e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:08:14+09:00", "last_error_summary": "[h264 @ 0x55a6a5e6ac00] error while decoding MB 2 27, bytestream -13 [in#0/rtsp @ 0x55a6a5638e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6a5638e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:08:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 229.912, "last_started_at": "2026-07-06T10:04:24+09:00", "pid": 2190, "process": {"exists": false, "pid": 2190}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.427, "sample_count": 1739, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 70.019}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1739, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:08:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T10:08:14+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-06T10:08:14+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T10:08:14.644794+09:00", "mtime_epoch": 1783300094.6447937, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211466}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:08:14+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1378.316, "name": "celery", "pid": 267, "rss_mb": 1369.656, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.103, "pushed": 2.103, "read": 2.07}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.145, "count": 63, "max_ms": 8.799, "p95_ms": 6.124}, "gstreamer_push": {"avg_ms": 1.409, "count": 63, "max_ms": 4.027, "p95_ms": 2.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.41, "count": 63, "max_ms": 73.17, "p95_ms": 65.488}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.111, "count": 63, "max_ms": 94.876, "p95_ms": 92.107}, "yolo11_640_fire_11class": {"avg_ms": 52.882, "count": 63, "max_ms": 73.991, "p95_ms": 66.072}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.641, "count": 63, "max_ms": 74.963, "p95_ms": 64.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.912, "count": 63, "max_ms": 146.975, "p95_ms": 136.709}, "read_interval": {"avg_ms": 483.026, "count": 62, "max_ms": 13275.316, "p95_ms": 157.168}, "triton_wait": {"avg_ms": 67.171, "count": 63, "max_ms": 94.888, "p95_ms": 92.121}, "visualize": {"avg_ms": 4.231, "count": 63, "max_ms": 6.583, "p95_ms": 5.637}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 14}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4611.172, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:08:14+09:00", "last_error_summary": "[h264 @ 0x55a6a5e6ac00] error while decoding MB 2 27, bytestream -13 [in#0/rtsp @ 0x55a6a5638e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6a5638e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:08:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 229.918, "last_started_at": "2026-07-06T10:04:24+09:00", "pid": 2190, "process": {"exists": false, "pid": 2190}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.427, "sample_count": 1739, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 70.013}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1739, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:08:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T10:08:14+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-06T10:08:14+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T10:08:14.644794+09:00", "mtime_epoch": 1783300094.6447937, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211466}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:08:14+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1378.316, "name": "celery", "pid": 267, "rss_mb": 1369.656, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.103, "pushed": 2.103, "read": 2.069}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.108, "count": 62, "max_ms": 8.799, "p95_ms": 5.814}, "gstreamer_push": {"avg_ms": 1.409, "count": 63, "max_ms": 4.027, "p95_ms": 2.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.422, "count": 62, "max_ms": 73.17, "p95_ms": 65.488}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.295, "count": 62, "max_ms": 94.876, "p95_ms": 92.107}, "yolo11_640_fire_11class": {"avg_ms": 52.905, "count": 62, "max_ms": 73.991, "p95_ms": 66.072}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.607, "count": 62, "max_ms": 74.963, "p95_ms": 64.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.912, "count": 63, "max_ms": 146.975, "p95_ms": 136.709}, "read_interval": {"avg_ms": 483.026, "count": 62, "max_ms": 13275.316, "p95_ms": 157.168}, "triton_wait": {"avg_ms": 67.357, "count": 62, "max_ms": 94.888, "p95_ms": 92.121}, "visualize": {"avg_ms": 4.216, "count": 62, "max_ms": 6.583, "p95_ms": 5.637}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 14}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4728.824, "limit_mb": 8796093022207.996}, "detail": {"pid": 2452, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:08:14+09:00", "last_error_summary": "[h264 @ 0x55a6a5e6ac00] error while decoding MB 2 27, bytestream -13 [in#0/rtsp @ 0x55a6a5638e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6a5638e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:08:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T10:08:17+09:00", "pid": 2452, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.262, "name": "ffmpeg", "pid": 2452, "rss_mb": 149.262, "state": "R (running)", "threads": 49}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.427, "sample_count": 1739, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 65.16}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1739, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:08:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 4.864, "last_frame_pushed_at": "2026-07-06T10:08:14+09:00", "last_frame_read_age_sec": 4.973, "last_frame_read_at": "2026-07-06T10:08:14+09:00", "mjpeg": {"age_sec": 4.844, "exists": true, "mtime": "2026-07-06T10:08:14.772795+09:00", "mtime_epoch": 1783300094.7727954, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211464}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:08:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1378.316, "name": "celery", "pid": 267, "rss_mb": 1369.656, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 18, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.633, "pushed": 0.633, "read": 0.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.713, "count": 19, "max_ms": 8.799, "p95_ms": 8.799}, "gstreamer_push": {"avg_ms": 1.583, "count": 19, "max_ms": 4.027, "p95_ms": 4.027}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.755, "count": 19, "max_ms": 73.17, "p95_ms": 73.17}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.705, "count": 19, "max_ms": 94.876, "p95_ms": 94.876}, "yolo11_640_fire_11class": {"avg_ms": 53.848, "count": 19, "max_ms": 73.991, "p95_ms": 73.991}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.07, "count": 19, "max_ms": 74.963, "p95_ms": 74.963}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.376, "count": 19, "max_ms": 146.975, "p95_ms": 146.975}, "read_interval": {"avg_ms": 1394.025, "count": 18, "max_ms": 13275.316, "p95_ms": 13275.316}, "triton_wait": {"avg_ms": 69.813, "count": 19, "max_ms": 94.888, "p95_ms": 94.888}, "visualize": {"avg_ms": 3.912, "count": 19, "max_ms": 6.153, "p95_ms": 6.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4624.625, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559e60120500] error while decoding MB 69 33, bytestream -5 [in#0/rtsp @ 0x559e5f81ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e5f81ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:14:09+09:00", "last_error_summary": "[h264 @ 0x559e60120500] error while decoding MB 69 33, bytestream -5 [in#0/rtsp @ 0x559e5f81ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e5f81ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:08:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 351.709, "last_started_at": "2026-07-06T10:08:17+09:00", "pid": 2452, "process": {"exists": false, "pid": 2452}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.558, "sample_count": 2465, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4667, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:14:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T10:14:09+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T10:14:09+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-06T10:14:09.269641+09:00", "mtime_epoch": 1783300449.2696412, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 205207}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:14:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1378.777, "name": "celery", "pid": 267, "rss_mb": 1369.605, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 44, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.47, "pushed": 1.47, "read": 1.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.283, "count": 44, "max_ms": 7.956, "p95_ms": 6.406}, "gstreamer_push": {"avg_ms": 1.661, "count": 44, "max_ms": 4.22, "p95_ms": 3.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.108, "count": 44, "max_ms": 70.773, "p95_ms": 67.563}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.763, "count": 44, "max_ms": 103.818, "p95_ms": 86.392}, "yolo11_640_fire_11class": {"avg_ms": 55.059, "count": 44, "max_ms": 71.159, "p95_ms": 68.57}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.944, "count": 44, "max_ms": 68.83, "p95_ms": 64.424}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.544, "count": 44, "max_ms": 140.461, "p95_ms": 132.095}, "read_interval": {"avg_ms": 681.184, "count": 44, "max_ms": 16819.86, "p95_ms": 150.137}, "triton_wait": {"avg_ms": 67.905, "count": 44, "max_ms": 103.835, "p95_ms": 86.404}, "visualize": {"avg_ms": 4.166, "count": 44, "max_ms": 6.104, "p95_ms": 5.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4624.078, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T10:14:09+09:00", "last_error_summary": "[h264 @ 0x559e60120500] error while decoding MB 69 33, bytestream -5 [in#0/rtsp @ 0x559e5f81ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e5f81ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:14:09+09:00", "last_error_summary": "[h264 @ 0x559e60120500] error while decoding MB 69 33, bytestream -5 [in#0/rtsp @ 0x559e5f81ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e5f81ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:14:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 351.732, "last_started_at": "2026-07-06T10:08:17+09:00", "pid": 2452, "process": {"exists": false, "pid": 2452}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.558, "sample_count": 2465, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:14:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T10:14:09+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-06T10:14:09+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-06T10:14:09.269641+09:00", "mtime_epoch": 1783300449.2696412, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 205207}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:14:09+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1378.777, "name": "celery", "pid": 267, "rss_mb": 1369.605, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.502, "pushed": 1.502, "read": 1.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.31, "count": 45, "max_ms": 7.956, "p95_ms": 6.406}, "gstreamer_push": {"avg_ms": 1.649, "count": 45, "max_ms": 4.22, "p95_ms": 3.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.891, "count": 45, "max_ms": 70.773, "p95_ms": 67.563}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.501, "count": 45, "max_ms": 103.818, "p95_ms": 86.392}, "yolo11_640_fire_11class": {"avg_ms": 54.914, "count": 45, "max_ms": 71.159, "p95_ms": 68.57}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.707, "count": 45, "max_ms": 68.83, "p95_ms": 64.424}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.206, "count": 45, "max_ms": 140.461, "p95_ms": 132.095}, "read_interval": {"avg_ms": 681.184, "count": 44, "max_ms": 16819.86, "p95_ms": 150.137}, "triton_wait": {"avg_ms": 67.64, "count": 45, "max_ms": 103.835, "p95_ms": 86.404}, "visualize": {"avg_ms": 4.262, "count": 45, "max_ms": 8.454, "p95_ms": 5.928}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4623.891, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:14:09+09:00", "last_error_summary": "[h264 @ 0x559e60120500] error while decoding MB 69 33, bytestream -5 [in#0/rtsp @ 0x559e5f81ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e5f81ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:14:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 351.738, "last_started_at": "2026-07-06T10:08:17+09:00", "pid": 2452, "process": {"exists": false, "pid": 2452}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.558, "sample_count": 2465, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:14:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T10:14:09+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-06T10:14:09+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-06T10:14:09.269641+09:00", "mtime_epoch": 1783300449.2696412, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 205207}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:14:09+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1378.777, "name": "celery", "pid": 267, "rss_mb": 1369.605, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.502, "pushed": 1.502, "read": 1.468}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.31, "count": 45, "max_ms": 7.956, "p95_ms": 6.406}, "gstreamer_push": {"avg_ms": 1.649, "count": 45, "max_ms": 4.22, "p95_ms": 3.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.891, "count": 45, "max_ms": 70.773, "p95_ms": 67.563}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.501, "count": 45, "max_ms": 103.818, "p95_ms": 86.392}, "yolo11_640_fire_11class": {"avg_ms": 54.914, "count": 45, "max_ms": 71.159, "p95_ms": 68.57}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.707, "count": 45, "max_ms": 68.83, "p95_ms": 64.424}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.206, "count": 45, "max_ms": 140.461, "p95_ms": 132.095}, "read_interval": {"avg_ms": 681.184, "count": 44, "max_ms": 16819.86, "p95_ms": 150.137}, "triton_wait": {"avg_ms": 67.64, "count": 45, "max_ms": 103.835, "p95_ms": 86.404}, "visualize": {"avg_ms": 4.262, "count": 45, "max_ms": 8.454, "p95_ms": 5.928}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4644.684, "limit_mb": 8796093022207.996}, "detail": {"pid": 2604, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:14:09+09:00", "last_error_summary": "[h264 @ 0x559e60120500] error while decoding MB 69 33, bytestream -5 [in#0/rtsp @ 0x559e5f81ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e5f81ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:14:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T10:14:11+09:00", "pid": 2604, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.246, "name": "ffmpeg", "pid": 2604, "rss_mb": 58.246, "state": "S (sleeping)", "threads": 1}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.652, "sample_count": 2423, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:14:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 4.219, "last_frame_pushed_at": "2026-07-06T10:14:09+09:00", "last_frame_read_age_sec": 4.296, "last_frame_read_at": "2026-07-06T10:14:09+09:00", "mjpeg": {"age_sec": 4.2, "exists": true, "mtime": "2026-07-06T10:14:09.357642+09:00", "mtime_epoch": 1783300449.3576424, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 204729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:14:13+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1378.777, "name": "celery", "pid": 267, "rss_mb": 1369.605, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.586, "pushed": 0.586, "read": 0.586}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.435, "count": 13, "max_ms": 6.406, "p95_ms": 6.406}, "gstreamer_push": {"avg_ms": 1.755, "count": 13, "max_ms": 3.629, "p95_ms": 3.629}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.018, "count": 13, "max_ms": 70.011, "p95_ms": 70.011}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.601, "count": 13, "max_ms": 76.756, "p95_ms": 76.756}, "yolo11_640_fire_11class": {"avg_ms": 53.824, "count": 13, "max_ms": 69.483, "p95_ms": 69.483}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.118, "count": 13, "max_ms": 62.216, "p95_ms": 62.216}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.543, "count": 13, "max_ms": 127.91, "p95_ms": 127.91}, "read_interval": {"avg_ms": 2056.858, "count": 13, "max_ms": 16819.86, "p95_ms": 16819.86}, "triton_wait": {"avg_ms": 62.614, "count": 13, "max_ms": 76.772, "p95_ms": 76.772}, "visualize": {"avg_ms": 4.706, "count": 13, "max_ms": 8.454, "p95_ms": 8.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.167, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 54}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4643.383, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a29a6e35c0] error while decoding MB 76 30, bytestream -35 [in#0/rtsp @ 0x55a29a6a6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a29a6a6e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:18:01+09:00", "last_error_summary": "[h264 @ 0x55a29a6e35c0] error while decoding MB 76 30, bytestream -35 [in#0/rtsp @ 0x55a29a6a6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a29a6a6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:14:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 230.24, "last_started_at": "2026-07-06T10:14:11+09:00", "pid": 2604, "process": {"exists": false, "pid": 2604}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.56, "sample_count": 2245, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6516, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:18:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T10:18:01+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T10:18:01+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T10:18:01.737059+09:00", "mtime_epoch": 1783300681.7370594, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232118}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:18:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1378.777, "name": "celery", "pid": 267, "rss_mb": 1375.988, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 61, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.039, "pushed": 2.039, "read": 2.039}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.194, "count": 61, "max_ms": 7.115, "p95_ms": 6.263}, "gstreamer_push": {"avg_ms": 1.328, "count": 61, "max_ms": 4.65, "p95_ms": 1.578}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.611, "count": 61, "max_ms": 74.485, "p95_ms": 72.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.382, "count": 61, "max_ms": 104.253, "p95_ms": 93.863}, "yolo11_640_fire_11class": {"avg_ms": 53.781, "count": 61, "max_ms": 73.214, "p95_ms": 70.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.615, "count": 61, "max_ms": 69.92, "p95_ms": 66.528}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.488, "count": 61, "max_ms": 155.753, "p95_ms": 142.329}, "read_interval": {"avg_ms": 491.036, "count": 61, "max_ms": 14635.55, "p95_ms": 161.122}, "triton_wait": {"avg_ms": 66.803, "count": 61, "max_ms": 104.266, "p95_ms": 93.875}, "visualize": {"avg_ms": 4.141, "count": 61, "max_ms": 5.696, "p95_ms": 5.48}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 54}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4642.996, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T10:18:01+09:00", "last_error_summary": "[h264 @ 0x55a29a6e35c0] error while decoding MB 76 30, bytestream -35 [in#0/rtsp @ 0x55a29a6a6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a29a6a6e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:18:01+09:00", "last_error_summary": "[h264 @ 0x55a29a6e35c0] error while decoding MB 76 30, bytestream -35 [in#0/rtsp @ 0x55a29a6a6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a29a6a6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:18:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 230.259, "last_started_at": "2026-07-06T10:14:11+09:00", "pid": 2604, "process": {"exists": false, "pid": 2604}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.56, "sample_count": 2246, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6517, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:18:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T10:18:01+09:00", "last_frame_read_age_sec": 0.081, "last_frame_read_at": "2026-07-06T10:18:01+09:00", "mjpeg": {"age_sec": 0.079, "exists": true, "mtime": "2026-07-06T10:18:01.737059+09:00", "mtime_epoch": 1783300681.7370594, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232118}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:18:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1378.777, "name": "celery", "pid": 267, "rss_mb": 1375.988, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.071, "pushed": 2.071, "read": 2.038}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.184, "count": 62, "max_ms": 7.115, "p95_ms": 6.263}, "gstreamer_push": {"avg_ms": 1.328, "count": 62, "max_ms": 4.65, "p95_ms": 1.578}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.475, "count": 62, "max_ms": 74.485, "p95_ms": 72.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.229, "count": 62, "max_ms": 104.253, "p95_ms": 93.863}, "yolo11_640_fire_11class": {"avg_ms": 53.658, "count": 62, "max_ms": 73.214, "p95_ms": 70.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.593, "count": 62, "max_ms": 69.92, "p95_ms": 66.528}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.194, "count": 62, "max_ms": 155.753, "p95_ms": 142.329}, "read_interval": {"avg_ms": 491.036, "count": 61, "max_ms": 14635.55, "p95_ms": 161.122}, "triton_wait": {"avg_ms": 66.643, "count": 62, "max_ms": 104.266, "p95_ms": 93.875}, "visualize": {"avg_ms": 4.176, "count": 62, "max_ms": 6.346, "p95_ms": 5.658}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 54}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4646.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:18:01+09:00", "last_error_summary": "[h264 @ 0x55a29a6e35c0] error while decoding MB 76 30, bytestream -35 [in#0/rtsp @ 0x55a29a6a6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a29a6a6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:18:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 230.267, "last_started_at": "2026-07-06T10:14:11+09:00", "pid": 2604, "process": {"exists": false, "pid": 2604}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.56, "sample_count": 2246, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6517, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:18:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T10:18:01+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-06T10:18:01+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-06T10:18:01.737059+09:00", "mtime_epoch": 1783300681.7370594, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232118}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:18:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1378.777, "name": "celery", "pid": 267, "rss_mb": 1375.988, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.071, "pushed": 2.071, "read": 2.037}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.184, "count": 62, "max_ms": 7.115, "p95_ms": 6.263}, "gstreamer_push": {"avg_ms": 1.328, "count": 62, "max_ms": 4.65, "p95_ms": 1.578}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.475, "count": 62, "max_ms": 74.485, "p95_ms": 72.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.229, "count": 62, "max_ms": 104.253, "p95_ms": 93.863}, "yolo11_640_fire_11class": {"avg_ms": 53.658, "count": 62, "max_ms": 73.214, "p95_ms": 70.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.593, "count": 62, "max_ms": 69.92, "p95_ms": 66.528}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.194, "count": 62, "max_ms": 155.753, "p95_ms": 142.329}, "read_interval": {"avg_ms": 491.036, "count": 61, "max_ms": 14635.55, "p95_ms": 161.122}, "triton_wait": {"avg_ms": 66.643, "count": 62, "max_ms": 104.266, "p95_ms": 93.875}, "visualize": {"avg_ms": 4.176, "count": 62, "max_ms": 6.346, "p95_ms": 5.658}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 54}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4831.465, "limit_mb": 8796093022207.996}, "detail": {"pid": 2756, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:18:01+09:00", "last_error_summary": "[h264 @ 0x55a29a6e35c0] error while decoding MB 76 30, bytestream -35 [in#0/rtsp @ 0x55a29a6a6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a29a6a6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:18:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T10:18:03+09:00", "pid": 2756, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.73, "name": "ffmpeg", "pid": 2756, "rss_mb": 206.73, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.572, "sample_count": 2200, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6517, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:18:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 4.094, "last_frame_pushed_at": "2026-07-06T10:18:01+09:00", "last_frame_read_age_sec": 4.169, "last_frame_read_at": "2026-07-06T10:18:01+09:00", "mjpeg": {"age_sec": 4.079, "exists": true, "mtime": "2026-07-06T10:18:01.825061+09:00", "mtime_epoch": 1783300681.8250606, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231961}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:18:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1378.777, "name": "celery", "pid": 267, "rss_mb": 1375.988, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 25, "pushed_frames": 25, "read_frames": 24, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.836, "pushed": 0.836, "read": 0.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.556, "count": 25, "max_ms": 5.671, "p95_ms": 4.555}, "gstreamer_push": {"avg_ms": 1.304, "count": 25, "max_ms": 1.578, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.118, "count": 25, "max_ms": 74.485, "p95_ms": 72.843}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.471, "count": 25, "max_ms": 93.863, "p95_ms": 90.656}, "yolo11_640_fire_11class": {"avg_ms": 52.647, "count": 25, "max_ms": 73.003, "p95_ms": 70.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.399, "count": 25, "max_ms": 69.92, "p95_ms": 67.694}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.726, "count": 25, "max_ms": 114.024, "p95_ms": 105.512}, "read_interval": {"avg_ms": 1076.319, "count": 24, "max_ms": 14635.55, "p95_ms": 9210.863}, "triton_wait": {"avg_ms": 64.92, "count": 25, "max_ms": 93.875, "p95_ms": 90.67}, "visualize": {"avg_ms": 3.79, "count": 25, "max_ms": 6.346, "p95_ms": 5.215}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 173}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4701.422, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55dce7b18140] error while decoding MB 38 32, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:44:32+09:00", "last_error_summary": "[h264 @ 0x55dce7b18140] error while decoding MB 38 32, bytestream -17", "last_exit_at": "2026-07-06T10:18:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 1588.607, "last_started_at": "2026-07-06T10:18:03+09:00", "pid": 2756, "process": {"exists": false, "pid": 2756}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.415, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20823, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:44:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T10:44:32+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T10:44:32+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-06T10:44:32.444770+09:00", "mtime_epoch": 1783302272.4447696, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239866}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:44:32+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1381.422, "name": "celery", "pid": 267, "rss_mb": 1381.414, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 246, "pushed_frames": 246, "read_frames": 245, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.205, "pushed": 8.205, "read": 8.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.362, "count": 245, "max_ms": 9.109, "p95_ms": 6.97}, "gstreamer_push": {"avg_ms": 1.287, "count": 246, "max_ms": 4.293, "p95_ms": 1.895}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.351, "count": 245, "max_ms": 74.958, "p95_ms": 68.915}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.707, "count": 245, "max_ms": 103.76, "p95_ms": 93.23}, "yolo11_640_fire_11class": {"avg_ms": 53.344, "count": 245, "max_ms": 73.757, "p95_ms": 68.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.684, "count": 245, "max_ms": 74.438, "p95_ms": 65.368}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.441, "count": 246, "max_ms": 178.674, "p95_ms": 146.01}, "read_interval": {"avg_ms": 122.683, "count": 245, "max_ms": 194.822, "p95_ms": 155.972}, "triton_wait": {"avg_ms": 68.937, "count": 245, "max_ms": 103.773, "p95_ms": 93.243}, "visualize": {"avg_ms": 4.808, "count": 245, "max_ms": 6.743, "p95_ms": 5.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 173}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4696.156, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T10:44:32+09:00", "last_error_summary": "[h264 @ 0x55dce7b18140] error while decoding MB 38 32, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:44:32+09:00", "last_error_summary": "[h264 @ 0x55dce7b18140] error while decoding MB 38 32, bytestream -17", "last_exit_at": "2026-07-06T10:44:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 1588.618, "last_started_at": "2026-07-06T10:18:03+09:00", "pid": 2756, "process": {"exists": false, "pid": 2756}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.415, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20823, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:44:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T10:44:32+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-06T10:44:32+09:00", "mjpeg": {"age_sec": 0.076, "exists": true, "mtime": "2026-07-06T10:44:32.444770+09:00", "mtime_epoch": 1783302272.4447696, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239866}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:44:32+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1381.422, "name": "celery", "pid": 267, "rss_mb": 1381.414, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 246, "pushed_frames": 246, "read_frames": 245, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.202, "pushed": 8.202, "read": 8.168}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.362, "count": 245, "max_ms": 9.109, "p95_ms": 6.97}, "gstreamer_push": {"avg_ms": 1.287, "count": 246, "max_ms": 4.293, "p95_ms": 1.895}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.351, "count": 245, "max_ms": 74.958, "p95_ms": 68.915}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.707, "count": 245, "max_ms": 103.76, "p95_ms": 93.23}, "yolo11_640_fire_11class": {"avg_ms": 53.344, "count": 245, "max_ms": 73.757, "p95_ms": 68.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.684, "count": 245, "max_ms": 74.438, "p95_ms": 65.368}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.441, "count": 246, "max_ms": 178.674, "p95_ms": 146.01}, "read_interval": {"avg_ms": 122.683, "count": 245, "max_ms": 194.822, "p95_ms": 155.972}, "triton_wait": {"avg_ms": 68.937, "count": 245, "max_ms": 103.773, "p95_ms": 93.243}, "visualize": {"avg_ms": 4.808, "count": 245, "max_ms": 6.743, "p95_ms": 5.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 173}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4698.164, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:44:32+09:00", "last_error_summary": "[h264 @ 0x55dce7b18140] error while decoding MB 38 32, bytestream -17", "last_exit_at": "2026-07-06T10:44:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 1588.627, "last_started_at": "2026-07-06T10:18:03+09:00", "pid": 2756, "process": {"exists": false, "pid": 2756}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.431, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20823, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:44:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.024, "last_frame_pushed_at": "2026-07-06T10:44:32+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-06T10:44:32+09:00", "mjpeg": {"age_sec": 0.008, "exists": true, "mtime": "2026-07-06T10:44:32.520771+09:00", "mtime_epoch": 1783302272.520771, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234674}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:44:32+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1381.422, "name": "celery", "pid": 267, "rss_mb": 1381.414, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 245, "pushed_frames": 245, "read_frames": 245, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.169, "pushed": 8.169, "read": 8.169}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.362, "count": 245, "max_ms": 9.109, "p95_ms": 6.97}, "gstreamer_push": {"avg_ms": 1.287, "count": 245, "max_ms": 4.293, "p95_ms": 1.895}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.351, "count": 245, "max_ms": 74.958, "p95_ms": 68.915}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.707, "count": 245, "max_ms": 103.76, "p95_ms": 93.23}, "yolo11_640_fire_11class": {"avg_ms": 53.344, "count": 245, "max_ms": 73.757, "p95_ms": 68.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.684, "count": 245, "max_ms": 74.438, "p95_ms": 65.368}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.327, "count": 245, "max_ms": 178.674, "p95_ms": 146.01}, "read_interval": {"avg_ms": 122.683, "count": 245, "max_ms": 194.822, "p95_ms": 155.972}, "triton_wait": {"avg_ms": 68.937, "count": 245, "max_ms": 103.773, "p95_ms": 93.243}, "visualize": {"avg_ms": 4.808, "count": 245, "max_ms": 6.743, "p95_ms": 5.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 173}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4870.461, "limit_mb": 8796093022207.996}, "detail": {"pid": 2936, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:44:32+09:00", "last_error_summary": "[h264 @ 0x55dce7b18140] error while decoding MB 38 32, bytestream -17", "last_exit_at": "2026-07-06T10:44:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T10:44:34+09:00", "pid": 2936, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.684, "name": "ffmpeg", "pid": 2936, "rss_mb": 197.684, "state": "S (sleeping)", "threads": 106}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.537, "sample_count": 2361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20823, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:44:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 4.41, "last_frame_pushed_at": "2026-07-06T10:44:32+09:00", "last_frame_read_age_sec": 4.47, "last_frame_read_at": "2026-07-06T10:44:32+09:00", "mjpeg": {"age_sec": 4.394, "exists": true, "mtime": "2026-07-06T10:44:32.520771+09:00", "mtime_epoch": 1783302272.520771, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234674}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:44:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1381.422, "name": "celery", "pid": 267, "rss_mb": 1381.414, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 211, "pushed_frames": 211, "read_frames": 210, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.056, "pushed": 7.056, "read": 7.023}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.271, "count": 211, "max_ms": 9.109, "p95_ms": 7.062}, "gstreamer_push": {"avg_ms": 1.314, "count": 211, "max_ms": 4.293, "p95_ms": 2.506}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.982, "count": 211, "max_ms": 74.958, "p95_ms": 69.301}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.161, "count": 211, "max_ms": 103.76, "p95_ms": 94.085}, "yolo11_640_fire_11class": {"avg_ms": 53.873, "count": 211, "max_ms": 73.757, "p95_ms": 68.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.734, "count": 211, "max_ms": 74.438, "p95_ms": 64.154}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.205, "count": 211, "max_ms": 178.674, "p95_ms": 146.393}, "read_interval": {"avg_ms": 121.603, "count": 210, "max_ms": 194.822, "p95_ms": 156.862}, "triton_wait": {"avg_ms": 69.348, "count": 211, "max_ms": 103.773, "p95_ms": 94.099}, "visualize": {"avg_ms": 4.762, "count": 211, "max_ms": 6.743, "p95_ms": 6.06}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 213}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4719.605, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x5604a85db200] missing picture in access unit with size 18 [h264 @ 0x5604a86546c0] no frame! [in#0/rtsp @ 0x5604a85d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5604a85d6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5604a85db840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:53:26+09:00", "last_error_summary": "[NULL @ 0x5604a85db200] missing picture in access unit with size 18 [h264 @ 0x5604a86546c0] no frame! [in#0/rtsp @ 0x5604a85d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5604a85d6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5604a85db840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T10:44:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 532.009, "last_started_at": "2026-07-06T10:44:34+09:00", "pid": 2936, "process": {"exists": false, "pid": 2936}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.747, "sample_count": 2545, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:53:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T10:53:26+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T10:53:26+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T10:53:26.871496+09:00", "mtime_epoch": 1783302806.8714964, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:53:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1386.664, "name": "celery", "pid": 267, "rss_mb": 1383.625, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.869, "pushed": 0.869, "read": 0.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.074, "count": 27, "max_ms": 8.341, "p95_ms": 5.921}, "gstreamer_push": {"avg_ms": 1.319, "count": 26, "max_ms": 1.757, "p95_ms": 1.535}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.651, "count": 27, "max_ms": 62.364, "p95_ms": 62.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.836, "count": 27, "max_ms": 88.425, "p95_ms": 82.471}, "yolo11_640_fire_11class": {"avg_ms": 50.574, "count": 27, "max_ms": 62.375, "p95_ms": 61.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.535, "count": 27, "max_ms": 67.689, "p95_ms": 63.081}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.598, "count": 26, "max_ms": 126.615, "p95_ms": 114.147}, "read_interval": {"avg_ms": 1152.617, "count": 26, "max_ms": 16733.516, "p95_ms": 10116.792}, "triton_wait": {"avg_ms": 60.224, "count": 27, "max_ms": 88.439, "p95_ms": 82.485}, "visualize": {"avg_ms": 3.994, "count": 27, "max_ms": 5.246, "p95_ms": 5.195}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 213}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4719.332, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T10:53:26+09:00", "last_error_summary": "[NULL @ 0x5604a85db200] missing picture in access unit with size 18 [h264 @ 0x5604a86546c0] no frame! [in#0/rtsp @ 0x5604a85d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5604a85d6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5604a85db840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:53:26+09:00", "last_error_summary": "[NULL @ 0x5604a85db200] missing picture in access unit with size 18 [h264 @ 0x5604a86546c0] no frame! [in#0/rtsp @ 0x5604a85d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5604a85d6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5604a85db840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T10:53:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 532.026, "last_started_at": "2026-07-06T10:44:34+09:00", "pid": 2936, "process": {"exists": false, "pid": 2936}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.747, "sample_count": 2546, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:53:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T10:53:26+09:00", "last_frame_read_age_sec": 0.081, "last_frame_read_at": "2026-07-06T10:53:26+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-06T10:53:26.871496+09:00", "mtime_epoch": 1783302806.8714964, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:53:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1386.664, "name": "celery", "pid": 267, "rss_mb": 1383.625, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 27, "pushed_frames": 27, "read_frames": 26, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.902, "pushed": 0.902, "read": 0.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.074, "count": 27, "max_ms": 8.341, "p95_ms": 5.921}, "gstreamer_push": {"avg_ms": 1.319, "count": 27, "max_ms": 1.757, "p95_ms": 1.535}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.651, "count": 27, "max_ms": 62.364, "p95_ms": 62.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.836, "count": 27, "max_ms": 88.425, "p95_ms": 82.471}, "yolo11_640_fire_11class": {"avg_ms": 50.574, "count": 27, "max_ms": 62.375, "p95_ms": 61.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.535, "count": 27, "max_ms": 67.689, "p95_ms": 63.081}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.334, "count": 27, "max_ms": 126.615, "p95_ms": 114.147}, "read_interval": {"avg_ms": 1152.617, "count": 26, "max_ms": 16733.516, "p95_ms": 10116.792}, "triton_wait": {"avg_ms": 60.224, "count": 27, "max_ms": 88.439, "p95_ms": 82.485}, "visualize": {"avg_ms": 3.994, "count": 27, "max_ms": 5.246, "p95_ms": 5.195}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 213}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4719.324, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:53:26+09:00", "last_error_summary": "[NULL @ 0x5604a85db200] missing picture in access unit with size 18 [h264 @ 0x5604a86546c0] no frame! [in#0/rtsp @ 0x5604a85d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5604a85d6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5604a85db840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T10:53:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 532.034, "last_started_at": "2026-07-06T10:44:34+09:00", "pid": 2936, "process": {"exists": false, "pid": 2936}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.747, "sample_count": 2546, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:53:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T10:53:26+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-06T10:53:26+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-06T10:53:26.871496+09:00", "mtime_epoch": 1783302806.8714964, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:53:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1386.664, "name": "celery", "pid": 267, "rss_mb": 1383.625, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 27, "pushed_frames": 27, "read_frames": 26, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.902, "pushed": 0.902, "read": 0.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.074, "count": 27, "max_ms": 8.341, "p95_ms": 5.921}, "gstreamer_push": {"avg_ms": 1.319, "count": 27, "max_ms": 1.757, "p95_ms": 1.535}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.651, "count": 27, "max_ms": 62.364, "p95_ms": 62.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.836, "count": 27, "max_ms": 88.425, "p95_ms": 82.471}, "yolo11_640_fire_11class": {"avg_ms": 50.574, "count": 27, "max_ms": 62.375, "p95_ms": 61.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.535, "count": 27, "max_ms": 67.689, "p95_ms": 63.081}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.334, "count": 27, "max_ms": 126.615, "p95_ms": 114.147}, "read_interval": {"avg_ms": 1152.617, "count": 26, "max_ms": 16733.516, "p95_ms": 10116.792}, "triton_wait": {"avg_ms": 60.224, "count": 27, "max_ms": 88.439, "p95_ms": 82.485}, "visualize": {"avg_ms": 3.994, "count": 27, "max_ms": 5.246, "p95_ms": 5.195}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 213}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4736.984, "limit_mb": 8796093022207.996}, "detail": {"pid": 3088, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T10:53:26+09:00", "last_error_summary": "[NULL @ 0x5604a85db200] missing picture in access unit with size 18 [h264 @ 0x5604a86546c0] no frame! [in#0/rtsp @ 0x5604a85d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5604a85d6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5604a85db840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T10:53:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T10:53:29+09:00", "pid": 3088, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.461, "name": "ffmpeg", "pid": 3088, "rss_mb": 57.461, "state": "S (sleeping)", "threads": 1}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.061, "sample_count": 2502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T10:53:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 4.312, "last_frame_pushed_at": "2026-07-06T10:53:26+09:00", "last_frame_read_age_sec": 4.387, "last_frame_read_at": "2026-07-06T10:53:26+09:00", "mjpeg": {"age_sec": 4.291, "exists": true, "mtime": "2026-07-06T10:53:26.955498+09:00", "mtime_epoch": 1783302806.955498, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239087}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T10:53:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1386.664, "name": "celery", "pid": 267, "rss_mb": 1383.625, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.543, "pushed": 0.543, "read": 0.543}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.061, "count": 12, "max_ms": 5.921, "p95_ms": 5.921}, "gstreamer_push": {"avg_ms": 1.313, "count": 12, "max_ms": 1.757, "p95_ms": 1.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.143, "count": 12, "max_ms": 62.364, "p95_ms": 62.364}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.752, "count": 12, "max_ms": 88.425, "p95_ms": 88.425}, "yolo11_640_fire_11class": {"avg_ms": 52.25, "count": 12, "max_ms": 62.375, "p95_ms": 62.375}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.093, "count": 12, "max_ms": 67.689, "p95_ms": 67.689}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.363, "count": 12, "max_ms": 126.615, "p95_ms": 126.615}, "read_interval": {"avg_ms": 2320.04, "count": 12, "max_ms": 16733.516, "p95_ms": 16733.516}, "triton_wait": {"avg_ms": 62.609, "count": 12, "max_ms": 88.439, "p95_ms": 88.439}, "visualize": {"avg_ms": 4.113, "count": 12, "max_ms": 5.246, "p95_ms": 5.246}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.111, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4739.676, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562f3a82ccc0] error while decoding MB 53 59, bytestream -5 [in#0/rtsp @ 0x562f39a45e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f39a45e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:09:27+09:00", "last_error_summary": "[h264 @ 0x562f3a82ccc0] error while decoding MB 53 59, bytestream -5 [in#0/rtsp @ 0x562f39a45e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f39a45e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T10:53:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 958.164, "last_started_at": "2026-07-06T10:53:29+09:00", "pid": 3088, "process": {"exists": false, "pid": 3088}, "restart_count": 5, "restart_count_per_hour": 4, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.748, "sample_count": 2597, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34786, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:09:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-06T11:09:27+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T11:09:27+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-06T11:09:27.342590+09:00", "mtime_epoch": 1783303767.3425896, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 212229}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:09:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1393.75, "name": "celery", "pid": 267, "rss_mb": 1382.809, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.681, "pushed": 0.681, "read": 0.737}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.437, "count": 12, "max_ms": 3.519, "p95_ms": 3.519}, "gstreamer_push": {"avg_ms": 1.331, "count": 12, "max_ms": 1.489, "p95_ms": 1.489}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.207, "count": 12, "max_ms": 75.223, "p95_ms": 75.223}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.959, "count": 12, "max_ms": 96.359, "p95_ms": 96.359}, "yolo11_640_fire_11class": {"avg_ms": 63.876, "count": 12, "max_ms": 73.752, "p95_ms": 73.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.216, "count": 12, "max_ms": 62.204, "p95_ms": 62.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.062, "count": 12, "max_ms": 109.065, "p95_ms": 109.065}, "read_interval": {"avg_ms": 2660.346, "count": 13, "max_ms": 17021.776, "p95_ms": 17021.776}, "triton_wait": {"avg_ms": 79.972, "count": 12, "max_ms": 96.372, "p95_ms": 96.372}, "visualize": {"avg_ms": 3.669, "count": 12, "max_ms": 4.026, "p95_ms": 4.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.628, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4744.113, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:09:27+09:00", "last_error_summary": "[h264 @ 0x562f3a82ccc0] error while decoding MB 53 59, bytestream -5 [in#0/rtsp @ 0x562f39a45e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f39a45e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:09:27+09:00", "last_error_summary": "[h264 @ 0x562f3a82ccc0] error while decoding MB 53 59, bytestream -5 [in#0/rtsp @ 0x562f39a45e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f39a45e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:09:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 958.199, "last_started_at": "2026-07-06T10:53:29+09:00", "pid": 3088, "process": {"exists": false, "pid": 3088}, "restart_count": 5, "restart_count_per_hour": 4, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.748, "sample_count": 2598, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34787, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:09:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T11:09:27+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-06T11:09:27+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T11:09:27.342590+09:00", "mtime_epoch": 1783303767.3425896, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 212229}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:09:27+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1393.75, "name": "celery", "pid": 267, "rss_mb": 1382.809, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.736, "pushed": 0.736, "read": 0.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.438, "count": 13, "max_ms": 3.519, "p95_ms": 3.519}, "gstreamer_push": {"avg_ms": 1.328, "count": 13, "max_ms": 1.489, "p95_ms": 1.489}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.032, "count": 13, "max_ms": 75.223, "p95_ms": 75.223}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.886, "count": 13, "max_ms": 96.359, "p95_ms": 96.359}, "yolo11_640_fire_11class": {"avg_ms": 63.843, "count": 13, "max_ms": 73.752, "p95_ms": 73.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.081, "count": 13, "max_ms": 62.204, "p95_ms": 62.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.019, "count": 13, "max_ms": 109.065, "p95_ms": 109.065}, "read_interval": {"avg_ms": 2660.346, "count": 13, "max_ms": 17021.776, "p95_ms": 17021.776}, "triton_wait": {"avg_ms": 79.899, "count": 13, "max_ms": 96.372, "p95_ms": 96.372}, "visualize": {"avg_ms": 3.667, "count": 13, "max_ms": 4.026, "p95_ms": 4.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.662, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4743.977, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:09:27+09:00", "last_error_summary": "[h264 @ 0x562f3a82ccc0] error while decoding MB 53 59, bytestream -5 [in#0/rtsp @ 0x562f39a45e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f39a45e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:09:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 958.206, "last_started_at": "2026-07-06T10:53:29+09:00", "pid": 3088, "process": {"exists": false, "pid": 3088}, "restart_count": 5, "restart_count_per_hour": 4, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.748, "sample_count": 2598, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34787, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:09:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T11:09:27+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-06T11:09:27+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T11:09:27.342590+09:00", "mtime_epoch": 1783303767.3425896, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 212229}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:09:27+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1393.75, "name": "celery", "pid": 267, "rss_mb": 1382.809, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.736, "pushed": 0.736, "read": 0.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.438, "count": 13, "max_ms": 3.519, "p95_ms": 3.519}, "gstreamer_push": {"avg_ms": 1.328, "count": 13, "max_ms": 1.489, "p95_ms": 1.489}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.032, "count": 13, "max_ms": 75.223, "p95_ms": 75.223}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.886, "count": 13, "max_ms": 96.359, "p95_ms": 96.359}, "yolo11_640_fire_11class": {"avg_ms": 63.843, "count": 13, "max_ms": 73.752, "p95_ms": 73.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.081, "count": 13, "max_ms": 62.204, "p95_ms": 62.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.019, "count": 13, "max_ms": 109.065, "p95_ms": 109.065}, "read_interval": {"avg_ms": 2660.346, "count": 13, "max_ms": 17021.776, "p95_ms": 17021.776}, "triton_wait": {"avg_ms": 79.899, "count": 13, "max_ms": 96.372, "p95_ms": 96.372}, "visualize": {"avg_ms": 3.667, "count": 13, "max_ms": 4.026, "p95_ms": 4.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.67, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4904.105, "limit_mb": 8796093022207.996}, "detail": {"pid": 3240, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:09:27+09:00", "last_error_summary": "[h264 @ 0x562f3a82ccc0] error while decoding MB 53 59, bytestream -5 [in#0/rtsp @ 0x562f39a45e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562f39a45e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:09:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:09:30+09:00", "pid": 3240, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.559, "name": "ffmpeg", "pid": 3240, "rss_mb": 197.559, "state": "S (sleeping)", "threads": 106}, "restart_count": 6, "restart_count_per_hour": 5, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.748, "sample_count": 2552, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34787, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:09:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 4.627, "last_frame_pushed_at": "2026-07-06T11:09:27+09:00", "last_frame_read_age_sec": 4.72, "last_frame_read_at": "2026-07-06T11:09:27+09:00", "mjpeg": {"age_sec": 4.603, "exists": true, "mtime": "2026-07-06T11:09:27.462592+09:00", "mtime_epoch": 1783303767.4625916, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 212139}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:09:32+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1393.75, "name": "celery", "pid": 267, "rss_mb": 1382.809, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.583, "pushed": 0.583, "read": 0.583}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.438, "count": 13, "max_ms": 3.519, "p95_ms": 3.519}, "gstreamer_push": {"avg_ms": 1.328, "count": 13, "max_ms": 1.489, "p95_ms": 1.489}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.032, "count": 13, "max_ms": 75.223, "p95_ms": 75.223}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.886, "count": 13, "max_ms": 96.359, "p95_ms": 96.359}, "yolo11_640_fire_11class": {"avg_ms": 63.843, "count": 13, "max_ms": 73.752, "p95_ms": 73.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.081, "count": 13, "max_ms": 62.204, "p95_ms": 62.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.019, "count": 13, "max_ms": 109.065, "p95_ms": 109.065}, "read_interval": {"avg_ms": 2660.346, "count": 13, "max_ms": 17021.776, "p95_ms": 17021.776}, "triton_wait": {"avg_ms": 79.899, "count": 13, "max_ms": 96.372, "p95_ms": 96.372}, "visualize": {"avg_ms": 3.667, "count": 13, "max_ms": 4.026, "p95_ms": 4.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.283, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4756.867, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557dd0174000] error while decoding MB 86 48, bytestream -5 [in#0/rtsp @ 0x557dd012ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557dd012ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:13:52+09:00", "last_error_summary": "[h264 @ 0x557dd0174000] error while decoding MB 86 48, bytestream -5 [in#0/rtsp @ 0x557dd012ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557dd012ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:09:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.828, "last_started_at": "2026-07-06T11:09:30+09:00", "pid": 3240, "process": {"exists": false, "pid": 3240}, "restart_count": 6, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.924, "sample_count": 2191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36965, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:13:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-06T11:13:52+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T11:13:52+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T11:13:52.842826+09:00", "mtime_epoch": 1783304032.842826, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218432}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:13:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1393.75, "name": "celery", "pid": 267, "rss_mb": 1388.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 27, "pushed_frames": 27, "read_frames": 27, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.901, "pushed": 0.901, "read": 0.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.147, "count": 27, "max_ms": 6.386, "p95_ms": 5.669}, "gstreamer_push": {"avg_ms": 1.319, "count": 27, "max_ms": 2.886, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.652, "count": 27, "max_ms": 69.601, "p95_ms": 68.96}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.376, "count": 27, "max_ms": 96.872, "p95_ms": 90.147}, "yolo11_640_fire_11class": {"avg_ms": 51.42, "count": 27, "max_ms": 69.544, "p95_ms": 68.034}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.8, "count": 27, "max_ms": 60.447, "p95_ms": 60.147}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.974, "count": 27, "max_ms": 127.152, "p95_ms": 123.73}, "read_interval": {"avg_ms": 1111.011, "count": 27, "max_ms": 16777.092, "p95_ms": 10692.513}, "triton_wait": {"avg_ms": 64.389, "count": 27, "max_ms": 96.889, "p95_ms": 90.162}, "visualize": {"avg_ms": 4.134, "count": 27, "max_ms": 5.513, "p95_ms": 5.149}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4756.82, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:13:52+09:00", "last_error_summary": "[h264 @ 0x557dd0174000] error while decoding MB 86 48, bytestream -5 [in#0/rtsp @ 0x557dd012ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557dd012ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:13:52+09:00", "last_error_summary": "[h264 @ 0x557dd0174000] error while decoding MB 86 48, bytestream -5 [in#0/rtsp @ 0x557dd012ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557dd012ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:13:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.852, "last_started_at": "2026-07-06T11:09:30+09:00", "pid": 3240, "process": {"exists": false, "pid": 3240}, "restart_count": 6, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.924, "sample_count": 2192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36966, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:13:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T11:13:52+09:00", "last_frame_read_age_sec": 0.087, "last_frame_read_at": "2026-07-06T11:13:52+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-06T11:13:52.842826+09:00", "mtime_epoch": 1783304032.842826, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218432}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:13:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1393.75, "name": "celery", "pid": 267, "rss_mb": 1388.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 28, "pushed_frames": 28, "read_frames": 27, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.934, "pushed": 0.934, "read": 0.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.289, "count": 27, "max_ms": 10.203, "p95_ms": 5.669}, "gstreamer_push": {"avg_ms": 1.318, "count": 28, "max_ms": 2.886, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.584, "count": 27, "max_ms": 69.601, "p95_ms": 68.96}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.283, "count": 27, "max_ms": 96.872, "p95_ms": 90.147}, "yolo11_640_fire_11class": {"avg_ms": 51.39, "count": 27, "max_ms": 69.544, "p95_ms": 68.034}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.136, "count": 27, "max_ms": 60.447, "p95_ms": 60.147}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.666, "count": 28, "max_ms": 127.152, "p95_ms": 123.73}, "read_interval": {"avg_ms": 1111.011, "count": 27, "max_ms": 16777.092, "p95_ms": 10692.513}, "triton_wait": {"avg_ms": 64.537, "count": 27, "max_ms": 96.889, "p95_ms": 90.162}, "visualize": {"avg_ms": 4.09, "count": 27, "max_ms": 5.513, "p95_ms": 5.139}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4756.754, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:13:52+09:00", "last_error_summary": "[h264 @ 0x557dd0174000] error while decoding MB 86 48, bytestream -5 [in#0/rtsp @ 0x557dd012ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557dd012ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:13:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.86, "last_started_at": "2026-07-06T11:09:30+09:00", "pid": 3240, "process": {"exists": false, "pid": 3240}, "restart_count": 6, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.924, "sample_count": 2192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36966, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:13:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T11:13:52+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-06T11:13:52+09:00", "mjpeg": {"age_sec": 0.005, "exists": true, "mtime": "2026-07-06T11:13:52.918827+09:00", "mtime_epoch": 1783304032.918827, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218367}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:13:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1393.75, "name": "celery", "pid": 267, "rss_mb": 1388.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 28, "pushed_frames": 28, "read_frames": 27, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.934, "pushed": 0.934, "read": 0.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.289, "count": 27, "max_ms": 10.203, "p95_ms": 5.669}, "gstreamer_push": {"avg_ms": 1.318, "count": 28, "max_ms": 2.886, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.584, "count": 27, "max_ms": 69.601, "p95_ms": 68.96}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.283, "count": 27, "max_ms": 96.872, "p95_ms": 90.147}, "yolo11_640_fire_11class": {"avg_ms": 51.39, "count": 27, "max_ms": 69.544, "p95_ms": 68.034}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.136, "count": 27, "max_ms": 60.447, "p95_ms": 60.147}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.666, "count": 28, "max_ms": 127.152, "p95_ms": 123.73}, "read_interval": {"avg_ms": 1111.011, "count": 27, "max_ms": 16777.092, "p95_ms": 10692.513}, "triton_wait": {"avg_ms": 64.537, "count": 27, "max_ms": 96.889, "p95_ms": 90.162}, "visualize": {"avg_ms": 4.09, "count": 27, "max_ms": 5.513, "p95_ms": 5.139}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_100424_c9313603.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4769.848, "limit_mb": 8796093022207.996}, "detail": {"pid": 3392, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:13:52+09:00", "last_error_summary": "[h264 @ 0x557dd0174000] error while decoding MB 86 48, bytestream -5 [in#0/rtsp @ 0x557dd012ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557dd012ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:13:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:13:55+09:00", "pid": 3392, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.84, "name": "ffmpeg", "pid": 3392, "rss_mb": 57.84, "state": "S (sleeping)", "threads": 1}, "restart_count": 7, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.924, "sample_count": 2192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36966, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:13:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "c9313603", "last_frame_pushed_age_sec": 4.486, "last_frame_pushed_at": "2026-07-06T11:13:52+09:00", "last_frame_read_age_sec": 4.568, "last_frame_read_at": "2026-07-06T11:13:52+09:00", "mjpeg": {"age_sec": 4.479, "exists": true, "mtime": "2026-07-06T11:13:52.918827+09:00", "mtime_epoch": 1783304032.918827, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218367}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:13:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1393.75, "name": "celery", "pid": 267, "rss_mb": 1388.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.623, "pushed": 0.623, "read": 0.623}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.033, "count": 14, "max_ms": 10.203, "p95_ms": 10.203}, "gstreamer_push": {"avg_ms": 1.44, "count": 14, "max_ms": 2.886, "p95_ms": 2.886}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.871, "count": 14, "max_ms": 69.601, "p95_ms": 69.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.51, "count": 14, "max_ms": 96.872, "p95_ms": 96.872}, "yolo11_640_fire_11class": {"avg_ms": 53.281, "count": 14, "max_ms": 69.544, "p95_ms": 69.544}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.798, "count": 14, "max_ms": 60.447, "p95_ms": 60.447}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.808, "count": 14, "max_ms": 108.934, "p95_ms": 108.934}, "read_interval": {"avg_ms": 2043.625, "count": 14, "max_ms": 16777.092, "p95_ms": 16777.092}, "triton_wait": {"avg_ms": 68.987, "count": 14, "max_ms": 96.889, "p95_ms": 96.889}, "visualize": {"avg_ms": 3.727, "count": 14, "max_ms": 4.115, "p95_ms": 4.115}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.486, "window_s": 30.0}, "started_at": "2026-07-06T10:04:24+09:00", "state": "running", "task_id": "777649da-7090-465f-838a-56540e888dcb", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_100440_8fc5102c.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 362}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4466.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x565019775340] error while decoding MB 48 2, bytestream -16", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:17:29+09:00", "last_error_summary": "[h264 @ 0x565019775340] error while decoding MB 48 2, bytestream -16", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 4369.068, "last_started_at": "2026-07-06T10:04:40+09:00", "pid": 2344, "process": {"exists": false, "pid": 2344}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.196, "sample_count": 3020, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43533, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:17:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "8fc5102c", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-06T11:17:29+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T11:17:29+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T11:17:29.366447+09:00", "mtime_epoch": 1783304249.3664467, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:17:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.523, "name": "celery", "pid": 237, "rss_mb": 881.445, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 320, "pushed_frames": 320, "read_frames": 319, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.678, "pushed": 10.678, "read": 10.644}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.355, "count": 320, "max_ms": 5.526, "p95_ms": 4.018}, "gstreamer_push": {"avg_ms": 1.588, "count": 320, "max_ms": 6.701, "p95_ms": 3.77}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.929, "count": 320, "max_ms": 71.209, "p95_ms": 65.056}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.307, "count": 320, "max_ms": 108.715, "p95_ms": 86.415}, "yolo11_640_fire_11class": {"avg_ms": 49.651, "count": 320, "max_ms": 72.767, "p95_ms": 64.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 37.485, "count": 320, "max_ms": 71.266, "p95_ms": 52.261}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.715, "count": 320, "max_ms": 121.378, "p95_ms": 98.778}, "read_interval": {"avg_ms": 93.809, "count": 319, "max_ms": 137.311, "p95_ms": 117.503}, "triton_wait": {"avg_ms": 64.404, "count": 320, "max_ms": 108.728, "p95_ms": 86.429}, "visualize": {"avg_ms": 3.75, "count": 320, "max_ms": 6.774, "p95_ms": 4.167}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-06T10:04:40+09:00", "state": "running", "task_id": "b7bb9bc3-2529-4776-bd26-c9fe21c510c6", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111751_2a3ad68f.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 2578.793, "limit_mb": 8796093022207.996}, "detail": {"pid": 272, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:17:51+09:00", "pid": 272, "process": {"exists": true, "fd_count": 5, "hwm_mb": 153.781, "name": "ffmpeg", "pid": 272, "rss_mb": 153.781, "state": "S (sleeping)", "threads": 49}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.881}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "warning", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["mjpeg_age_sec>10s"]}, "instance_id": "2a3ad68f", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 14.388, "exists": true, "mtime": "2026-07-06T11:17:39.266615+09:00", "mtime_epoch": 1783304259.266615, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 208829}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:17:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 467.738, "name": "celery", "pid": 267, "rss_mb": 467.738, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_warning"], "recommended_action": "degrade_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T11:17:51+09:00", "state": "running", "task_id": "e797adc6-7859-4a46-af41-be18fcbb250c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 2778.355, "limit_mb": 8796093022207.996}, "detail": {"pid": 544, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:18:00+09:00", "pid": 544, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.227, "name": "ffmpeg", "pid": 544, "rss_mb": 58.227, "state": "S (sleeping)", "threads": 1}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.936}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 2.757, "exists": true, "mtime": "2026-07-06T11:17:59.546960+09:00", "mtime_epoch": 1783304279.5469599, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 199665}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:18:02+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 861.945, "name": "celery", "pid": 267, "rss_mb": 756.992, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3160.695, "limit_mb": 8796093022207.996}, "detail": {"pid": 549, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:18:01+09:00", "pid": 549, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.344, "name": "ffmpeg", "pid": 549, "rss_mb": 139.344, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.901}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 34.624, "exists": true, "mtime": "2026-07-06T11:17:29.366447+09:00", "mtime_epoch": 1783304249.3664467, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:18:03+09:00", "process": {"exists": true, "fd_count": 106, "hwm_mb": 467.645, "name": "celery", "pid": 237, "rss_mb": 467.645, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260706_111806_5100b9bf.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 3976.82, "limit_mb": 8796093022207.996}, "detail": {"pid": 821, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:18:06+09:00", "pid": 821, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.617, "name": "ffmpeg", "pid": 821, "rss_mb": 137.617, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.733}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "5100b9bf", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 39.456, "exists": true, "mtime": "2026-07-06T11:17:29.338446+09:00", "mtime_epoch": 1783304249.3384464, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 264943}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-06T11:18:08+09:00", "process": {"exists": true, "fd_count": 110, "hwm_mb": 470.746, "name": "celery", "pid": 239, "rss_mb": 470.746, "state": "R (running)", "threads": 7}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T11:18:06+09:00", "state": "running", "task_id": "ff23f075-7244-425e-bc50-224b67e2a14d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4170.234, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563e4c1df900] error while decoding MB 66 21, bytestream -17 [in#0/rtsp @ 0x563e4b80ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e4b80ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:21:42+09:00", "last_error_summary": "[h264 @ 0x563e4c1df900] error while decoding MB 66 21, bytestream -17 [in#0/rtsp @ 0x563e4b80ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e4b80ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 222.658, "last_started_at": "2026-07-06T11:18:00+09:00", "pid": 544, "process": {"exists": false, "pid": 544}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.382, "sample_count": 1629, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 77.278}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1629, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:21:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T11:21:42+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-06T11:21:42+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T11:21:42.910804+09:00", "mtime_epoch": 1783304502.910804, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:21:42+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1044.473, "name": "celery", "pid": 267, "rss_mb": 1040.867, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.12, "pushed": 1.12, "read": 1.206}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.061, "count": 13, "max_ms": 3.434, "p95_ms": 3.434}, "gstreamer_push": {"avg_ms": 1.37, "count": 13, "max_ms": 1.479, "p95_ms": 1.479}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.223, "count": 13, "max_ms": 87.42, "p95_ms": 87.42}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 89.02, "count": 13, "max_ms": 115.522, "p95_ms": 115.522}, "yolo11_640_fire_11class": {"avg_ms": 71.02, "count": 13, "max_ms": 91.145, "p95_ms": 91.145}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.122, "count": 13, "max_ms": 106.119, "p95_ms": 106.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.48, "count": 13, "max_ms": 127.252, "p95_ms": 127.252}, "read_interval": {"avg_ms": 2159.24, "count": 14, "max_ms": 18678.822, "p95_ms": 18678.822}, "triton_wait": {"avg_ms": 89.545, "count": 13, "max_ms": 115.535, "p95_ms": 115.535}, "visualize": {"avg_ms": 3.857, "count": 13, "max_ms": 5.006, "p95_ms": 5.006}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.61, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4167.109, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:21:42+09:00", "last_error_summary": "[h264 @ 0x563e4c1df900] error while decoding MB 66 21, bytestream -17 [in#0/rtsp @ 0x563e4b80ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e4b80ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:21:42+09:00", "last_error_summary": "[h264 @ 0x563e4c1df900] error while decoding MB 66 21, bytestream -17 [in#0/rtsp @ 0x563e4b80ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e4b80ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:21:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 222.766, "last_started_at": "2026-07-06T11:18:00+09:00", "pid": 544, "process": {"exists": false, "pid": 544}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.455, "sample_count": 1630, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 77.171}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:21:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T11:21:43+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-06T11:21:42+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-06T11:21:42.910804+09:00", "mtime_epoch": 1783304502.910804, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:21:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1044.512, "name": "celery", "pid": 267, "rss_mb": 1038.129, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.195, "pushed": 1.195, "read": 1.195}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.295, "count": 14, "max_ms": 6.349, "p95_ms": 6.349}, "gstreamer_push": {"avg_ms": 1.352, "count": 14, "max_ms": 1.479, "p95_ms": 1.479}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.15, "count": 14, "max_ms": 87.42, "p95_ms": 87.42}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 89.513, "count": 14, "max_ms": 115.522, "p95_ms": 115.522}, "yolo11_640_fire_11class": {"avg_ms": 71.361, "count": 14, "max_ms": 91.145, "p95_ms": 91.145}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.725, "count": 14, "max_ms": 106.119, "p95_ms": 106.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.739, "count": 14, "max_ms": 161.111, "p95_ms": 161.111}, "read_interval": {"avg_ms": 2159.24, "count": 14, "max_ms": 18678.822, "p95_ms": 18678.822}, "triton_wait": {"avg_ms": 90.001, "count": 14, "max_ms": 115.535, "p95_ms": 115.535}, "visualize": {"avg_ms": 3.959, "count": 14, "max_ms": 5.278, "p95_ms": 5.278}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.718, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4167.094, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:21:42+09:00", "last_error_summary": "[h264 @ 0x563e4c1df900] error while decoding MB 66 21, bytestream -17 [in#0/rtsp @ 0x563e4b80ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e4b80ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:21:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 222.772, "last_started_at": "2026-07-06T11:18:00+09:00", "pid": 544, "process": {"exists": false, "pid": 544}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.455, "sample_count": 1630, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 77.165}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:21:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T11:21:43+09:00", "last_frame_read_age_sec": 0.173, "last_frame_read_at": "2026-07-06T11:21:42+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-06T11:21:42.910804+09:00", "mtime_epoch": 1783304502.910804, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:21:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1044.512, "name": "celery", "pid": 267, "rss_mb": 1038.129, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.194, "pushed": 1.194, "read": 1.194}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.295, "count": 14, "max_ms": 6.349, "p95_ms": 6.349}, "gstreamer_push": {"avg_ms": 1.352, "count": 14, "max_ms": 1.479, "p95_ms": 1.479}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.15, "count": 14, "max_ms": 87.42, "p95_ms": 87.42}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 89.513, "count": 14, "max_ms": 115.522, "p95_ms": 115.522}, "yolo11_640_fire_11class": {"avg_ms": 71.361, "count": 14, "max_ms": 91.145, "p95_ms": 91.145}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.725, "count": 14, "max_ms": 106.119, "p95_ms": 106.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.739, "count": 14, "max_ms": 161.111, "p95_ms": 161.111}, "read_interval": {"avg_ms": 2159.24, "count": 14, "max_ms": 18678.822, "p95_ms": 18678.822}, "triton_wait": {"avg_ms": 90.001, "count": 14, "max_ms": 115.535, "p95_ms": 115.535}, "visualize": {"avg_ms": 3.959, "count": 14, "max_ms": 5.278, "p95_ms": 5.278}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.724, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4346.727, "limit_mb": 8796093022207.996}, "detail": {"pid": 1088, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:21:42+09:00", "last_error_summary": "[h264 @ 0x563e4c1df900] error while decoding MB 66 21, bytestream -17 [in#0/rtsp @ 0x563e4b80ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e4b80ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:21:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:21:46+09:00", "pid": 1088, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.637, "name": "ffmpeg", "pid": 1088, "rss_mb": 209.637, "state": "S (sleeping)", "threads": 106}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.455, "sample_count": 1630, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 72.194}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:21:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 4.981, "last_frame_pushed_at": "2026-07-06T11:21:43+09:00", "last_frame_read_age_sec": 5.144, "last_frame_read_at": "2026-07-06T11:21:42+09:00", "mjpeg": {"age_sec": 4.967, "exists": true, "mtime": "2026-07-06T11:21:43.078807+09:00", "mtime_epoch": 1783304503.078807, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222702}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:21:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1044.512, "name": "celery", "pid": 267, "rss_mb": 1038.129, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.839, "pushed": 0.839, "read": 0.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.295, "count": 14, "max_ms": 6.349, "p95_ms": 6.349}, "gstreamer_push": {"avg_ms": 1.352, "count": 14, "max_ms": 1.479, "p95_ms": 1.479}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.15, "count": 14, "max_ms": 87.42, "p95_ms": 87.42}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 89.513, "count": 14, "max_ms": 115.522, "p95_ms": 115.522}, "yolo11_640_fire_11class": {"avg_ms": 71.361, "count": 14, "max_ms": 91.145, "p95_ms": 91.145}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.725, "count": 14, "max_ms": 106.119, "p95_ms": 106.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.739, "count": 14, "max_ms": 161.111, "p95_ms": 161.111}, "read_interval": {"avg_ms": 2159.24, "count": 14, "max_ms": 18678.822, "p95_ms": 18678.822}, "triton_wait": {"avg_ms": 90.001, "count": 14, "max_ms": 115.535, "p95_ms": 115.535}, "visualize": {"avg_ms": 3.959, "count": 14, "max_ms": 5.278, "p95_ms": 5.278}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.695, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 29}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4241.738, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55598e1c0840] error while decoding MB 6 29, bytestream -20", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:24:53+09:00", "last_error_summary": "[h264 @ 0x55598e1c0840] error while decoding MB 6 29, bytestream -20", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 411.612, "last_started_at": "2026-07-06T11:18:01+09:00", "pid": 549, "process": {"exists": false, "pid": 549}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 186.537, "sample_count": 2484, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3493, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:24:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T11:24:53+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T11:24:53+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T11:24:53.558135+09:00", "mtime_epoch": 1783304693.5581346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200844}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:24:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 860.328, "name": "celery", "pid": 237, "rss_mb": 854.477, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.962, "pushed": 4.962, "read": 4.962}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.271, "count": 148, "max_ms": 13.685, "p95_ms": 10.315}, "gstreamer_push": {"avg_ms": 0.93, "count": 148, "max_ms": 2.011, "p95_ms": 1.065}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.13, "count": 148, "max_ms": 82.103, "p95_ms": 74.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.913, "count": 148, "max_ms": 122.572, "p95_ms": 107.06}, "yolo11_640_fire_11class": {"avg_ms": 59.365, "count": 148, "max_ms": 89.86, "p95_ms": 79.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.252, "count": 148, "max_ms": 110.246, "p95_ms": 99.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.907, "count": 148, "max_ms": 272.023, "p95_ms": 233.749}, "read_interval": {"avg_ms": 202.441, "count": 148, "max_ms": 279.609, "p95_ms": 240.952}, "triton_wait": {"avg_ms": 78.276, "count": 148, "max_ms": 122.586, "p95_ms": 107.073}, "visualize": {"avg_ms": 8.108, "count": 148, "max_ms": 10.87, "p95_ms": 9.533}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.827, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 29}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4241.152, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:24:53+09:00", "last_error_summary": "[h264 @ 0x55598e1c0840] error while decoding MB 6 29, bytestream -20", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:24:53+09:00", "last_error_summary": "[h264 @ 0x55598e1c0840] error while decoding MB 6 29, bytestream -20", "last_exit_at": "2026-07-06T11:24:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 411.758, "last_started_at": "2026-07-06T11:18:01+09:00", "pid": 549, "process": {"exists": false, "pid": 549}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 186.733, "sample_count": 2484, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:24:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T11:24:53+09:00", "last_frame_read_age_sec": 0.199, "last_frame_read_at": "2026-07-06T11:24:53+09:00", "mjpeg": {"age_sec": 0.189, "exists": true, "mtime": "2026-07-06T11:24:53.558135+09:00", "mtime_epoch": 1783304693.5581346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200844}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:24:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.328, "name": "celery", "pid": 237, "rss_mb": 854.52, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.971, "pushed": 4.971, "read": 4.938}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.311, "count": 148, "max_ms": 14.293, "p95_ms": 10.376}, "gstreamer_push": {"avg_ms": 0.929, "count": 149, "max_ms": 2.011, "p95_ms": 1.065}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.96, "count": 148, "max_ms": 82.103, "p95_ms": 74.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.773, "count": 148, "max_ms": 122.572, "p95_ms": 107.06}, "yolo11_640_fire_11class": {"avg_ms": 59.257, "count": 148, "max_ms": 89.86, "p95_ms": 79.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.084, "count": 148, "max_ms": 110.246, "p95_ms": 99.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.901, "count": 149, "max_ms": 272.023, "p95_ms": 233.749}, "read_interval": {"avg_ms": 202.441, "count": 148, "max_ms": 279.609, "p95_ms": 240.952}, "triton_wait": {"avg_ms": 78.119, "count": 148, "max_ms": 122.586, "p95_ms": 107.073}, "visualize": {"avg_ms": 8.114, "count": 148, "max_ms": 10.87, "p95_ms": 9.533}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 29}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4241.066, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:24:53+09:00", "last_error_summary": "[h264 @ 0x55598e1c0840] error while decoding MB 6 29, bytestream -20", "last_exit_at": "2026-07-06T11:24:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 411.762, "last_started_at": "2026-07-06T11:18:01+09:00", "pid": 549, "process": {"exists": false, "pid": 549}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 186.733, "sample_count": 2484, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:24:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T11:24:53+09:00", "last_frame_read_age_sec": 0.203, "last_frame_read_at": "2026-07-06T11:24:53+09:00", "mjpeg": {"age_sec": 0.193, "exists": true, "mtime": "2026-07-06T11:24:53.558135+09:00", "mtime_epoch": 1783304693.5581346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200844}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:24:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.328, "name": "celery", "pid": 237, "rss_mb": 854.52, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.97, "pushed": 4.97, "read": 4.937}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.311, "count": 148, "max_ms": 14.293, "p95_ms": 10.376}, "gstreamer_push": {"avg_ms": 0.929, "count": 149, "max_ms": 2.011, "p95_ms": 1.065}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.96, "count": 148, "max_ms": 82.103, "p95_ms": 74.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.773, "count": 148, "max_ms": 122.572, "p95_ms": 107.06}, "yolo11_640_fire_11class": {"avg_ms": 59.257, "count": 148, "max_ms": 89.86, "p95_ms": 79.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.084, "count": 148, "max_ms": 110.246, "p95_ms": 99.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.901, "count": 149, "max_ms": 272.023, "p95_ms": 233.749}, "read_interval": {"avg_ms": 202.441, "count": 148, "max_ms": 279.609, "p95_ms": 240.952}, "triton_wait": {"avg_ms": 78.119, "count": 148, "max_ms": 122.586, "p95_ms": 107.073}, "visualize": {"avg_ms": 8.114, "count": 148, "max_ms": 10.87, "p95_ms": 9.533}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 29}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4255.117, "limit_mb": 8796093022207.996}, "detail": {"pid": 1350, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:24:53+09:00", "last_error_summary": "[h264 @ 0x55598e1c0840] error while decoding MB 6 29, bytestream -20", "last_exit_at": "2026-07-06T11:24:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:24:55+09:00", "pid": 1350, "process": {"exists": true, "fd_count": 4, "hwm_mb": 46.801, "name": "ffmpeg", "pid": 1350, "rss_mb": 46.801, "state": "S (sleeping)", "threads": 1}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.19, "sample_count": 2446, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:24:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.229, "last_frame_pushed_at": "2026-07-06T11:24:53+09:00", "last_frame_read_age_sec": 4.425, "last_frame_read_at": "2026-07-06T11:24:53+09:00", "mjpeg": {"age_sec": 4.211, "exists": true, "mtime": "2026-07-06T11:24:53.762138+09:00", "mtime_epoch": 1783304693.7621384, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201003}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:24:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 860.328, "name": "celery", "pid": 237, "rss_mb": 854.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.292, "pushed": 4.292, "read": 4.259}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.333, "count": 128, "max_ms": 14.293, "p95_ms": 10.49}, "gstreamer_push": {"avg_ms": 0.939, "count": 128, "max_ms": 2.011, "p95_ms": 1.073}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.664, "count": 128, "max_ms": 82.103, "p95_ms": 74.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.164, "count": 128, "max_ms": 122.572, "p95_ms": 107.552}, "yolo11_640_fire_11class": {"avg_ms": 59.178, "count": 128, "max_ms": 89.86, "p95_ms": 79.468}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.729, "count": 128, "max_ms": 110.246, "p95_ms": 99.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.985, "count": 128, "max_ms": 272.023, "p95_ms": 233.749}, "read_interval": {"avg_ms": 201.515, "count": 127, "max_ms": 279.609, "p95_ms": 240.952}, "triton_wait": {"avg_ms": 77.429, "count": 128, "max_ms": 122.586, "p95_ms": 107.566}, "visualize": {"avg_ms": 8.085, "count": 128, "max_ms": 10.87, "p95_ms": 9.533}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.82, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4249.273, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:27:08+09:00", "last_error_summary": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "last_exit_at": "2026-07-06T11:24:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.608, "last_started_at": "2026-07-06T11:24:55+09:00", "pid": 1350, "process": {"exists": false, "pid": 1350}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 201.657, "sample_count": 1932, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4244, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:27:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T11:27:08+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T11:27:08+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T11:27:08.540512+09:00", "mtime_epoch": 1783304828.5405118, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199021}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:27:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 860.938, "name": "celery", "pid": 237, "rss_mb": 855.121, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.345, "pushed": 5.345, "read": 5.345}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.931, "count": 159, "max_ms": 13.393, "p95_ms": 9.956}, "gstreamer_push": {"avg_ms": 1.024, "count": 160, "max_ms": 4.334, "p95_ms": 1.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.256, "count": 159, "max_ms": 86.858, "p95_ms": 75.724}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.699, "count": 159, "max_ms": 115.304, "p95_ms": 107.156}, "yolo11_640_fire_11class": {"avg_ms": 61.863, "count": 159, "max_ms": 89.955, "p95_ms": 82.013}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.421, "count": 159, "max_ms": 109.565, "p95_ms": 101.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 180.201, "count": 160, "max_ms": 229.201, "p95_ms": 210.264}, "read_interval": {"avg_ms": 187.76, "count": 160, "max_ms": 237.253, "p95_ms": 217.847}, "triton_wait": {"avg_ms": 79.433, "count": 159, "max_ms": 115.318, "p95_ms": 107.44}, "visualize": {"avg_ms": 7.585, "count": 159, "max_ms": 11.617, "p95_ms": 9.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4241.664, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:27:08+09:00", "last_error_summary": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:27:08+09:00", "last_error_summary": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "last_exit_at": "2026-07-06T11:27:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.767, "last_started_at": "2026-07-06T11:24:55+09:00", "pid": 1350, "process": {"exists": false, "pid": 1350}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 201.932, "sample_count": 1932, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4245, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:27:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T11:27:08+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-06T11:27:08+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-06T11:27:08.540512+09:00", "mtime_epoch": 1783304828.5405118, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199021}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:27:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.938, "name": "celery", "pid": 237, "rss_mb": 848.875, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.341, "pushed": 5.341, "read": 5.308}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.935, "count": 159, "max_ms": 13.393, "p95_ms": 9.956}, "gstreamer_push": {"avg_ms": 1.03, "count": 160, "max_ms": 4.334, "p95_ms": 1.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.346, "count": 159, "max_ms": 86.858, "p95_ms": 75.724}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.023, "count": 159, "max_ms": 115.304, "p95_ms": 107.156}, "yolo11_640_fire_11class": {"avg_ms": 61.998, "count": 159, "max_ms": 89.955, "p95_ms": 82.013}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.567, "count": 159, "max_ms": 109.565, "p95_ms": 101.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 180.498, "count": 160, "max_ms": 229.201, "p95_ms": 210.264}, "read_interval": {"avg_ms": 187.918, "count": 159, "max_ms": 237.253, "p95_ms": 218.444}, "triton_wait": {"avg_ms": 79.749, "count": 159, "max_ms": 115.318, "p95_ms": 107.44}, "visualize": {"avg_ms": 7.589, "count": 159, "max_ms": 11.617, "p95_ms": 9.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4241.656, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:27:08+09:00", "last_error_summary": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "last_exit_at": "2026-07-06T11:27:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.771, "last_started_at": "2026-07-06T11:24:55+09:00", "pid": 1350, "process": {"exists": false, "pid": 1350}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 201.932, "sample_count": 1932, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4245, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:27:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T11:27:08+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-06T11:27:08+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-06T11:27:08.540512+09:00", "mtime_epoch": 1783304828.5405118, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199021}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:27:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.938, "name": "celery", "pid": 237, "rss_mb": 848.875, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.341, "pushed": 5.341, "read": 5.307}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.935, "count": 159, "max_ms": 13.393, "p95_ms": 9.956}, "gstreamer_push": {"avg_ms": 1.03, "count": 160, "max_ms": 4.334, "p95_ms": 1.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.346, "count": 159, "max_ms": 86.858, "p95_ms": 75.724}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.023, "count": 159, "max_ms": 115.304, "p95_ms": 107.156}, "yolo11_640_fire_11class": {"avg_ms": 61.998, "count": 159, "max_ms": 89.955, "p95_ms": 82.013}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.567, "count": 159, "max_ms": 109.565, "p95_ms": 101.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 180.498, "count": 160, "max_ms": 229.201, "p95_ms": 210.264}, "read_interval": {"avg_ms": 187.918, "count": 159, "max_ms": 237.253, "p95_ms": 218.444}, "triton_wait": {"avg_ms": 79.749, "count": 159, "max_ms": 115.318, "p95_ms": 107.44}, "visualize": {"avg_ms": 7.589, "count": 159, "max_ms": 11.617, "p95_ms": 9.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4344.648, "limit_mb": 8796093022207.996}, "detail": {"pid": 1457, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:27:08+09:00", "last_error_summary": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "last_exit_at": "2026-07-06T11:27:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:27:11+09:00", "pid": 1457, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.145, "name": "ffmpeg", "pid": 1457, "rss_mb": 139.145, "state": "S (sleeping)", "threads": 106}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 202.213, "sample_count": 1884, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4245, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:27:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.817, "last_frame_pushed_at": "2026-07-06T11:27:08+09:00", "last_frame_read_age_sec": 5.023, "last_frame_read_at": "2026-07-06T11:27:08+09:00", "mjpeg": {"age_sec": 4.801, "exists": true, "mtime": "2026-07-06T11:27:08.752516+09:00", "mtime_epoch": 1783304828.7525156, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192626}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:27:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 860.938, "name": "celery", "pid": 237, "rss_mb": 848.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.367, "pushed": 4.4, "read": 4.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.133, "count": 131, "max_ms": 13.393, "p95_ms": 9.97}, "gstreamer_push": {"avg_ms": 1.007, "count": 132, "max_ms": 3.269, "p95_ms": 1.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.917, "count": 131, "max_ms": 86.858, "p95_ms": 75.724}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.314, "count": 131, "max_ms": 115.304, "p95_ms": 107.428}, "yolo11_640_fire_11class": {"avg_ms": 62.959, "count": 131, "max_ms": 89.955, "p95_ms": 82.705}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.514, "count": 131, "max_ms": 109.565, "p95_ms": 101.962}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.829, "count": 131, "max_ms": 229.201, "p95_ms": 211.852}, "read_interval": {"avg_ms": 192.228, "count": 131, "max_ms": 237.253, "p95_ms": 218.444}, "triton_wait": {"avg_ms": 80.181, "count": 131, "max_ms": 115.318, "p95_ms": 107.519}, "visualize": {"avg_ms": 7.83, "count": 131, "max_ms": 11.617, "p95_ms": 10.01}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 40}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4469.438, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:27:08+09:00", "last_error_summary": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:27:08+09:00", "last_error_summary": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "last_exit_at": "2026-07-06T11:29:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.192, "last_started_at": "2026-07-06T11:27:11+09:00", "pid": 1457, "process": {"exists": false, "pid": 1457}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.91, "sample_count": 1531, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4823, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:29:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T11:29:13+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-06T11:29:13+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-06T11:29:13.546724+09:00", "mtime_epoch": 1783304953.5467238, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192835}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:29:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1069.016, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 163, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.49, "pushed": 5.49, "read": 5.456}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.213, "count": 164, "max_ms": 12.214, "p95_ms": 9.574}, "gstreamer_push": {"avg_ms": 0.93, "count": 164, "max_ms": 1.169, "p95_ms": 1.024}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.103, "count": 164, "max_ms": 86.319, "p95_ms": 77.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.664, "count": 164, "max_ms": 121.322, "p95_ms": 105.274}, "yolo11_640_fire_11class": {"avg_ms": 56.611, "count": 164, "max_ms": 85.391, "p95_ms": 77.835}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.595, "count": 164, "max_ms": 106.187, "p95_ms": 101.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 176.275, "count": 164, "max_ms": 223.357, "p95_ms": 209.481}, "read_interval": {"avg_ms": 183.2, "count": 163, "max_ms": 232.79, "p95_ms": 215.845}, "triton_wait": {"avg_ms": 74.913, "count": 164, "max_ms": 121.334, "p95_ms": 106.126}, "visualize": {"avg_ms": 7.623, "count": 164, "max_ms": 10.396, "p95_ms": 8.614}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 40}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4469.508, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:27:08+09:00", "last_error_summary": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "last_exit_at": "2026-07-06T11:29:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.195, "last_started_at": "2026-07-06T11:27:11+09:00", "pid": 1457, "process": {"exists": false, "pid": 1457}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.91, "sample_count": 1531, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4823, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:29:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T11:29:13+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-06T11:29:13+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-06T11:29:13.546724+09:00", "mtime_epoch": 1783304953.5467238, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192835}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:29:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1069.016, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 163, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.489, "pushed": 5.489, "read": 5.456}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.213, "count": 164, "max_ms": 12.214, "p95_ms": 9.574}, "gstreamer_push": {"avg_ms": 0.93, "count": 164, "max_ms": 1.169, "p95_ms": 1.024}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.103, "count": 164, "max_ms": 86.319, "p95_ms": 77.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.664, "count": 164, "max_ms": 121.322, "p95_ms": 105.274}, "yolo11_640_fire_11class": {"avg_ms": 56.611, "count": 164, "max_ms": 85.391, "p95_ms": 77.835}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.595, "count": 164, "max_ms": 106.187, "p95_ms": 101.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 176.275, "count": 164, "max_ms": 223.357, "p95_ms": 209.481}, "read_interval": {"avg_ms": 183.2, "count": 163, "max_ms": 232.79, "p95_ms": 215.845}, "triton_wait": {"avg_ms": 74.913, "count": 164, "max_ms": 121.334, "p95_ms": 106.126}, "visualize": {"avg_ms": 7.623, "count": 164, "max_ms": 10.396, "p95_ms": 8.614}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 40}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4567.48, "limit_mb": 8796093022207.996}, "detail": {"pid": 1625, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:27:08+09:00", "last_error_summary": "[h264 @ 0x563fbac08e40] error while decoding MB 62 1, bytestream -16", "last_exit_at": "2026-07-06T11:29:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:29:15+09:00", "pid": 1625, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.52, "name": "ffmpeg", "pid": 1625, "rss_mb": 136.52, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.725, "sample_count": 1509, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4823, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:29:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.147, "last_frame_pushed_at": "2026-07-06T11:29:13+09:00", "last_frame_read_age_sec": 4.353, "last_frame_read_at": "2026-07-06T11:29:13+09:00", "mjpeg": {"age_sec": 4.13, "exists": true, "mtime": "2026-07-06T11:29:13.754728+09:00", "mtime_epoch": 1783304953.7547276, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193043}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:29:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1069.016, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.716, "pushed": 4.716, "read": 4.683}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.132, "count": 141, "max_ms": 12.214, "p95_ms": 9.386}, "gstreamer_push": {"avg_ms": 0.932, "count": 141, "max_ms": 1.169, "p95_ms": 1.025}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.717, "count": 141, "max_ms": 86.319, "p95_ms": 78.146}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.146, "count": 141, "max_ms": 121.322, "p95_ms": 107.489}, "yolo11_640_fire_11class": {"avg_ms": 57.249, "count": 141, "max_ms": 85.391, "p95_ms": 77.873}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.212, "count": 141, "max_ms": 106.187, "p95_ms": 101.611}}, "truncated_count": 0}, "processing_total": {"avg_ms": 176.636, "count": 141, "max_ms": 223.357, "p95_ms": 209.754}, "read_interval": {"avg_ms": 183.579, "count": 140, "max_ms": 232.79, "p95_ms": 216.554}, "triton_wait": {"avg_ms": 75.434, "count": 141, "max_ms": 121.334, "p95_ms": 107.502}, "visualize": {"avg_ms": 7.625, "count": 141, "max_ms": 10.396, "p95_ms": 8.614}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 45}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4469.168, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559cbfe4bb00] error while decoding MB 58 38, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:31:21+09:00", "last_error_summary": "[h264 @ 0x559cbfe4bb00] error while decoding MB 58 38, bytestream -7", "last_exit_at": "2026-07-06T11:29:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.506, "last_started_at": "2026-07-06T11:29:15+09:00", "pid": 1625, "process": {"exists": false, "pid": 1625}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.684, "sample_count": 1513, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5493, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:31:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T11:31:21+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T11:31:21+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-06T11:31:21.344994+09:00", "mtime_epoch": 1783305081.3449936, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193402}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:31:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 870.348, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 165, "pushed_frames": 165, "read_frames": 165, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.527, "pushed": 5.527, "read": 5.527}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.261, "count": 165, "max_ms": 13.706, "p95_ms": 10.703}, "gstreamer_push": {"avg_ms": 0.952, "count": 165, "max_ms": 2.138, "p95_ms": 1.117}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.343, "count": 165, "max_ms": 84.777, "p95_ms": 73.709}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.475, "count": 165, "max_ms": 127.815, "p95_ms": 104.46}, "yolo11_640_fire_11class": {"avg_ms": 56.521, "count": 165, "max_ms": 85.748, "p95_ms": 80.647}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.61, "count": 165, "max_ms": 115.052, "p95_ms": 101.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.777, "count": 165, "max_ms": 220.941, "p95_ms": 210.168}, "read_interval": {"avg_ms": 181.923, "count": 165, "max_ms": 230.007, "p95_ms": 216.094}, "triton_wait": {"avg_ms": 71.986, "count": 165, "max_ms": 127.824, "p95_ms": 104.661}, "visualize": {"avg_ms": 7.678, "count": 165, "max_ms": 11.561, "p95_ms": 9.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.853, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 45}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4468.613, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:31:21+09:00", "last_error_summary": "[h264 @ 0x559cbfe4bb00] error while decoding MB 58 38, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:31:21+09:00", "last_error_summary": "[h264 @ 0x559cbfe4bb00] error while decoding MB 58 38, bytestream -7", "last_exit_at": "2026-07-06T11:31:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.626, "last_started_at": "2026-07-06T11:29:15+09:00", "pid": 1625, "process": {"exists": false, "pid": 1625}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.684, "sample_count": 1514, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:31:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T11:31:21+09:00", "last_frame_read_age_sec": 0.177, "last_frame_read_at": "2026-07-06T11:31:21+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-06T11:31:21.344994+09:00", "mtime_epoch": 1783305081.3449936, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193402}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:31:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 870.352, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 166, "pushed_frames": 166, "read_frames": 165, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.538, "pushed": 5.538, "read": 5.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.291, "count": 165, "max_ms": 13.706, "p95_ms": 11.021}, "gstreamer_push": {"avg_ms": 0.952, "count": 166, "max_ms": 2.138, "p95_ms": 1.117}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.189, "count": 165, "max_ms": 84.777, "p95_ms": 73.709}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.152, "count": 165, "max_ms": 127.815, "p95_ms": 104.46}, "yolo11_640_fire_11class": {"avg_ms": 56.29, "count": 165, "max_ms": 85.748, "p95_ms": 80.647}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.305, "count": 165, "max_ms": 115.052, "p95_ms": 101.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.758, "count": 166, "max_ms": 220.941, "p95_ms": 210.168}, "read_interval": {"avg_ms": 181.923, "count": 165, "max_ms": 230.007, "p95_ms": 216.094}, "triton_wait": {"avg_ms": 71.662, "count": 165, "max_ms": 127.824, "p95_ms": 104.661}, "visualize": {"avg_ms": 7.691, "count": 165, "max_ms": 11.561, "p95_ms": 9.322}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 45}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4468.605, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:31:21+09:00", "last_error_summary": "[h264 @ 0x559cbfe4bb00] error while decoding MB 58 38, bytestream -7", "last_exit_at": "2026-07-06T11:31:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.63, "last_started_at": "2026-07-06T11:29:15+09:00", "pid": 1625, "process": {"exists": false, "pid": 1625}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.684, "sample_count": 1514, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:31:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T11:31:21+09:00", "last_frame_read_age_sec": 0.18, "last_frame_read_at": "2026-07-06T11:31:21+09:00", "mjpeg": {"age_sec": 0.169, "exists": true, "mtime": "2026-07-06T11:31:21.344994+09:00", "mtime_epoch": 1783305081.3449936, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193402}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:31:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 870.352, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 166, "pushed_frames": 166, "read_frames": 165, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.538, "pushed": 5.538, "read": 5.504}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.291, "count": 165, "max_ms": 13.706, "p95_ms": 11.021}, "gstreamer_push": {"avg_ms": 0.952, "count": 166, "max_ms": 2.138, "p95_ms": 1.117}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.189, "count": 165, "max_ms": 84.777, "p95_ms": 73.709}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.152, "count": 165, "max_ms": 127.815, "p95_ms": 104.46}, "yolo11_640_fire_11class": {"avg_ms": 56.29, "count": 165, "max_ms": 85.748, "p95_ms": 80.647}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.305, "count": 165, "max_ms": 115.052, "p95_ms": 101.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.758, "count": 166, "max_ms": 220.941, "p95_ms": 210.168}, "read_interval": {"avg_ms": 181.923, "count": 165, "max_ms": 230.007, "p95_ms": 216.094}, "triton_wait": {"avg_ms": 71.662, "count": 165, "max_ms": 127.824, "p95_ms": 104.661}, "visualize": {"avg_ms": 7.691, "count": 165, "max_ms": 11.561, "p95_ms": 9.322}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 45}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4568.918, "limit_mb": 8796093022207.996}, "detail": {"pid": 1793, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:31:21+09:00", "last_error_summary": "[h264 @ 0x559cbfe4bb00] error while decoding MB 58 38, bytestream -7", "last_exit_at": "2026-07-06T11:31:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:31:23+09:00", "pid": 1793, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.262, "name": "ffmpeg", "pid": 1793, "rss_mb": 139.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.255, "sample_count": 1490, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:31:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.066, "last_frame_pushed_at": "2026-07-06T11:31:21+09:00", "last_frame_read_age_sec": 4.24, "last_frame_read_at": "2026-07-06T11:31:21+09:00", "mjpeg": {"age_sec": 4.052, "exists": true, "mtime": "2026-07-06T11:31:21.520997+09:00", "mtime_epoch": 1783305081.5209966, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193655}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:31:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 870.352, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.797, "pushed": 4.797, "read": 4.763}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.347, "count": 143, "max_ms": 13.706, "p95_ms": 11.06}, "gstreamer_push": {"avg_ms": 0.958, "count": 143, "max_ms": 2.138, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.598, "count": 143, "max_ms": 84.777, "p95_ms": 70.983}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.692, "count": 143, "max_ms": 127.815, "p95_ms": 104.46}, "yolo11_640_fire_11class": {"avg_ms": 55.849, "count": 143, "max_ms": 85.748, "p95_ms": 79.441}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.947, "count": 143, "max_ms": 115.052, "p95_ms": 102.888}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.344, "count": 143, "max_ms": 219.289, "p95_ms": 209.308}, "read_interval": {"avg_ms": 181.505, "count": 142, "max_ms": 228.317, "p95_ms": 215.496}, "triton_wait": {"avg_ms": 71.267, "count": 143, "max_ms": 127.824, "p95_ms": 104.661}, "visualize": {"avg_ms": 7.735, "count": 143, "max_ms": 11.561, "p95_ms": 9.322}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.811, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 51}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4396.766, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:33:35+09:00", "last_error_summary": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "last_exit_at": "2026-07-06T11:31:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.61, "last_started_at": "2026-07-06T11:31:23+09:00", "pid": 1793, "process": {"exists": false, "pid": 1793}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 210.332, "sample_count": 1592, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6204, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:33:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T11:33:35+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-06T11:33:35+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T11:33:35.139376+09:00", "mtime_epoch": 1783305215.1393764, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:33:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 870.234, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 162, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.416, "pushed": 5.416, "read": 5.416}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.255, "count": 162, "max_ms": 12.604, "p95_ms": 10.249}, "gstreamer_push": {"avg_ms": 1.05, "count": 162, "max_ms": 3.707, "p95_ms": 2.06}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.228, "count": 162, "max_ms": 85.214, "p95_ms": 78.369}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.706, "count": 162, "max_ms": 127.043, "p95_ms": 109.191}, "yolo11_640_fire_11class": {"avg_ms": 60.127, "count": 162, "max_ms": 91.685, "p95_ms": 84.054}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.626, "count": 162, "max_ms": 110.954, "p95_ms": 103.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 178.01, "count": 162, "max_ms": 249.928, "p95_ms": 219.796}, "read_interval": {"avg_ms": 185.256, "count": 162, "max_ms": 259.245, "p95_ms": 229.069}, "triton_wait": {"avg_ms": 75.357, "count": 162, "max_ms": 127.057, "p95_ms": 109.213}, "visualize": {"avg_ms": 7.564, "count": 162, "max_ms": 10.205, "p95_ms": 9.233}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 51}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4395.996, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:33:35+09:00", "last_error_summary": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:33:35+09:00", "last_error_summary": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "last_exit_at": "2026-07-06T11:33:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.761, "last_started_at": "2026-07-06T11:31:23+09:00", "pid": 1793, "process": {"exists": false, "pid": 1793}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 210.332, "sample_count": 1593, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:33:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T11:33:35+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-06T11:33:35+09:00", "mjpeg": {"age_sec": 0.194, "exists": true, "mtime": "2026-07-06T11:33:35.139376+09:00", "mtime_epoch": 1783305215.1393764, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:33:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 870.234, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 161, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.418, "pushed": 5.418, "read": 5.385}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.25, "count": 162, "max_ms": 12.604, "p95_ms": 10.249}, "gstreamer_push": {"avg_ms": 1.051, "count": 162, "max_ms": 3.707, "p95_ms": 2.06}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.441, "count": 162, "max_ms": 85.214, "p95_ms": 78.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.982, "count": 162, "max_ms": 127.043, "p95_ms": 109.191}, "yolo11_640_fire_11class": {"avg_ms": 60.287, "count": 162, "max_ms": 91.685, "p95_ms": 84.054}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.886, "count": 162, "max_ms": 110.954, "p95_ms": 103.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 178.292, "count": 162, "max_ms": 249.928, "p95_ms": 219.796}, "read_interval": {"avg_ms": 185.372, "count": 161, "max_ms": 259.245, "p95_ms": 229.069}, "triton_wait": {"avg_ms": 75.625, "count": 162, "max_ms": 127.057, "p95_ms": 109.213}, "visualize": {"avg_ms": 7.561, "count": 162, "max_ms": 10.205, "p95_ms": 9.233}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 51}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4395.988, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:33:35+09:00", "last_error_summary": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "last_exit_at": "2026-07-06T11:33:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.765, "last_started_at": "2026-07-06T11:31:23+09:00", "pid": 1793, "process": {"exists": false, "pid": 1793}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 210.332, "sample_count": 1592, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:33:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T11:33:35+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-06T11:33:35+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-06T11:33:35.139376+09:00", "mtime_epoch": 1783305215.1393764, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:33:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 870.234, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 161, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.418, "pushed": 5.418, "read": 5.384}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.25, "count": 162, "max_ms": 12.604, "p95_ms": 10.249}, "gstreamer_push": {"avg_ms": 1.051, "count": 162, "max_ms": 3.707, "p95_ms": 2.06}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.441, "count": 162, "max_ms": 85.214, "p95_ms": 78.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.982, "count": 162, "max_ms": 127.043, "p95_ms": 109.191}, "yolo11_640_fire_11class": {"avg_ms": 60.287, "count": 162, "max_ms": 91.685, "p95_ms": 84.054}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.886, "count": 162, "max_ms": 110.954, "p95_ms": 103.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 178.292, "count": 162, "max_ms": 249.928, "p95_ms": 219.796}, "read_interval": {"avg_ms": 185.372, "count": 161, "max_ms": 259.245, "p95_ms": 229.069}, "triton_wait": {"avg_ms": 75.625, "count": 162, "max_ms": 127.057, "p95_ms": 109.213}, "visualize": {"avg_ms": 7.561, "count": 162, "max_ms": 10.205, "p95_ms": 9.233}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 51}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4506.75, "limit_mb": 8796093022207.996}, "detail": {"pid": 1900, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:33:35+09:00", "last_error_summary": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "last_exit_at": "2026-07-06T11:33:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:33:37+09:00", "pid": 1900, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.418, "name": "ffmpeg", "pid": 1900, "rss_mb": 137.418, "state": "S (sleeping)", "threads": 106}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 210.332, "sample_count": 1568, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:33:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.49, "last_frame_pushed_at": "2026-07-06T11:33:35+09:00", "last_frame_read_age_sec": 4.698, "last_frame_read_at": "2026-07-06T11:33:35+09:00", "mjpeg": {"age_sec": 4.473, "exists": true, "mtime": "2026-07-06T11:33:35.347380+09:00", "mtime_epoch": 1783305215.3473802, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195593}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:33:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 870.234, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.609, "pushed": 4.609, "read": 4.575}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.263, "count": 137, "max_ms": 12.604, "p95_ms": 10.267}, "gstreamer_push": {"avg_ms": 1.061, "count": 138, "max_ms": 3.707, "p95_ms": 2.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.043, "count": 137, "max_ms": 85.214, "p95_ms": 79.907}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.527, "count": 137, "max_ms": 127.043, "p95_ms": 109.191}, "yolo11_640_fire_11class": {"avg_ms": 60.616, "count": 137, "max_ms": 91.685, "p95_ms": 84.864}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.503, "count": 137, "max_ms": 110.954, "p95_ms": 105.201}}, "truncated_count": 0}, "processing_total": {"avg_ms": 178.532, "count": 138, "max_ms": 249.928, "p95_ms": 222.725}, "read_interval": {"avg_ms": 185.552, "count": 137, "max_ms": 259.245, "p95_ms": 230.072}, "triton_wait": {"avg_ms": 76.155, "count": 137, "max_ms": 127.057, "p95_ms": 109.213}, "visualize": {"avg_ms": 7.543, "count": 137, "max_ms": 10.205, "p95_ms": 9.297}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4509.266, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:33:35+09:00", "last_error_summary": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:33:35+09:00", "last_error_summary": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "last_exit_at": "2026-07-06T11:35:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.215, "last_started_at": "2026-07-06T11:33:37+09:00", "pid": 1900, "process": {"exists": false, "pid": 1900}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.809, "sample_count": 1407, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6641, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:35:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T11:35:34+09:00", "last_frame_read_age_sec": 0.281, "last_frame_read_at": "2026-07-06T11:35:33+09:00", "mjpeg": {"age_sec": 0.269, "exists": true, "mtime": "2026-07-06T11:35:33.765399+09:00", "mtime_epoch": 1783305333.7653995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195645}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:35:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.016, "pushed": 4.016, "read": 3.983}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.345, "count": 119, "max_ms": 14.908, "p95_ms": 12.607}, "gstreamer_push": {"avg_ms": 0.864, "count": 120, "max_ms": 1.188, "p95_ms": 0.919}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.241, "count": 119, "max_ms": 78.137, "p95_ms": 73.683}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.292, "count": 119, "max_ms": 116.575, "p95_ms": 109.954}, "yolo11_640_fire_11class": {"avg_ms": 60.95, "count": 119, "max_ms": 91.711, "p95_ms": 83.033}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.555, "count": 119, "max_ms": 113.268, "p95_ms": 106.181}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.99, "count": 120, "max_ms": 283.608, "p95_ms": 278.246}, "read_interval": {"avg_ms": 250.664, "count": 119, "max_ms": 291.869, "p95_ms": 286.071}, "triton_wait": {"avg_ms": 82.014, "count": 119, "max_ms": 116.588, "p95_ms": 111.045}, "visualize": {"avg_ms": 10.409, "count": 119, "max_ms": 12.182, "p95_ms": 11.601}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4509.258, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:33:35+09:00", "last_error_summary": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "last_exit_at": "2026-07-06T11:35:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.218, "last_started_at": "2026-07-06T11:33:37+09:00", "pid": 1900, "process": {"exists": false, "pid": 1900}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.809, "sample_count": 1407, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6641, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:35:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T11:35:34+09:00", "last_frame_read_age_sec": 0.284, "last_frame_read_at": "2026-07-06T11:35:33+09:00", "mjpeg": {"age_sec": 0.272, "exists": true, "mtime": "2026-07-06T11:35:33.765399+09:00", "mtime_epoch": 1783305333.7653995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195645}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:35:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.016, "pushed": 4.016, "read": 3.982}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.345, "count": 119, "max_ms": 14.908, "p95_ms": 12.607}, "gstreamer_push": {"avg_ms": 0.864, "count": 120, "max_ms": 1.188, "p95_ms": 0.919}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.241, "count": 119, "max_ms": 78.137, "p95_ms": 73.683}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.292, "count": 119, "max_ms": 116.575, "p95_ms": 109.954}, "yolo11_640_fire_11class": {"avg_ms": 60.95, "count": 119, "max_ms": 91.711, "p95_ms": 83.033}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.555, "count": 119, "max_ms": 113.268, "p95_ms": 106.181}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.99, "count": 120, "max_ms": 283.608, "p95_ms": 278.246}, "read_interval": {"avg_ms": 250.664, "count": 119, "max_ms": 291.869, "p95_ms": 286.071}, "triton_wait": {"avg_ms": 82.014, "count": 119, "max_ms": 116.588, "p95_ms": 111.045}, "visualize": {"avg_ms": 10.409, "count": 119, "max_ms": 12.182, "p95_ms": 11.601}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4611.672, "limit_mb": 8796093022207.996}, "detail": {"pid": 2067, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:33:35+09:00", "last_error_summary": "[h264 @ 0x55fcf74c0b00] error while decoding MB 50 35, bytestream -9", "last_exit_at": "2026-07-06T11:35:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:35:36+09:00", "pid": 2067, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.418, "name": "ffmpeg", "pid": 2067, "rss_mb": 139.418, "state": "S (sleeping)", "threads": 106}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.085, "sample_count": 1383, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6641, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:35:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.383, "last_frame_pushed_at": "2026-07-06T11:35:34+09:00", "last_frame_read_age_sec": 4.659, "last_frame_read_at": "2026-07-06T11:35:33+09:00", "mjpeg": {"age_sec": 4.367, "exists": true, "mtime": "2026-07-06T11:35:34.045404+09:00", "mtime_epoch": 1783305334.0454042, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196064}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:35:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.066, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.442, "pushed": 3.442, "read": 3.408}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.3, "count": 102, "max_ms": 14.908, "p95_ms": 12.554}, "gstreamer_push": {"avg_ms": 0.864, "count": 103, "max_ms": 1.188, "p95_ms": 0.919}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.061, "count": 102, "max_ms": 78.137, "p95_ms": 73.66}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.107, "count": 102, "max_ms": 116.575, "p95_ms": 109.431}, "yolo11_640_fire_11class": {"avg_ms": 60.588, "count": 102, "max_ms": 91.711, "p95_ms": 80.699}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.583, "count": 102, "max_ms": 113.268, "p95_ms": 106.017}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.346, "count": 103, "max_ms": 283.608, "p95_ms": 275.923}, "read_interval": {"avg_ms": 249.971, "count": 102, "max_ms": 291.869, "p95_ms": 283.257}, "triton_wait": {"avg_ms": 81.888, "count": 102, "max_ms": 116.588, "p95_ms": 111.045}, "visualize": {"avg_ms": 10.391, "count": 102, "max_ms": 12.182, "p95_ms": 11.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 60}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4521.125, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a1ab885580] cabac decode of qscale diff failed at 37 41 [h264 @ 0x55a1ab885580] error while decoding MB 37 41, bytestream 34956 [h264 @ 0x55a1ac372d40] error while decoding MB 20 22, bytestream -9 [h264 @ 0x55a1ac2a3ec0] cabac decode of qscale diff failed at 57 13 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 57 13, bytestream 46425 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 28 24, bytestream -9 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 47 38, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:37:55+09:00", "last_error_summary": "[h264 @ 0x55a1ab885580] cabac decode of qscale diff failed at 37 41 [h264 @ 0x55a1ab885580] error while decoding MB 37 41, bytestream 34956 [h264 @ 0x55a1ac372d40] error while decoding MB 20 22, bytestream -9 [h264 @ 0x55a1ac2a3ec0] cabac decode of qscale diff failed at 57 13 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 57 13, bytestream 46425 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 28 24, bytestream -9 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 47 38, bytestream -12", "last_exit_at": "2026-07-06T11:35:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.591, "last_started_at": "2026-07-06T11:35:36+09:00", "pid": 2067, "process": {"exists": false, "pid": 2067}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.967, "sample_count": 1317, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:37:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.043, "last_frame_pushed_at": "2026-07-06T11:37:54+09:00", "last_frame_read_age_sec": 0.039, "last_frame_read_at": "2026-07-06T11:37:54+09:00", "mjpeg": {"age_sec": 0.027, "exists": true, "mtime": "2026-07-06T11:37:54.975799+09:00", "mtime_epoch": 1783305474.9757986, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186793}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:37:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.098, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.91, "pushed": 4.91, "read": 4.91}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.585, "count": 146, "max_ms": 15.0, "p95_ms": 11.59}, "gstreamer_push": {"avg_ms": 0.89, "count": 147, "max_ms": 1.113, "p95_ms": 0.998}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.095, "count": 146, "max_ms": 83.439, "p95_ms": 74.457}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.417, "count": 146, "max_ms": 116.45, "p95_ms": 108.108}, "yolo11_640_fire_11class": {"avg_ms": 58.549, "count": 146, "max_ms": 85.753, "p95_ms": 78.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.616, "count": 146, "max_ms": 111.171, "p95_ms": 99.926}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.825, "count": 147, "max_ms": 250.566, "p95_ms": 235.855}, "read_interval": {"avg_ms": 204.912, "count": 147, "max_ms": 257.13, "p95_ms": 244.15}, "triton_wait": {"avg_ms": 77.804, "count": 146, "max_ms": 116.462, "p95_ms": 108.438}, "visualize": {"avg_ms": 8.551, "count": 146, "max_ms": 11.642, "p95_ms": 10.655}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 60}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4513.879, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:37:55+09:00", "last_error_summary": "[h264 @ 0x55a1ab885580] cabac decode of qscale diff failed at 37 41 [h264 @ 0x55a1ab885580] error while decoding MB 37 41, bytestream 34956 [h264 @ 0x55a1ac372d40] error while decoding MB 20 22, bytestream -9 [h264 @ 0x55a1ac2a3ec0] cabac decode of qscale diff failed at 57 13 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 57 13, bytestream 46425 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 28 24, bytestream -9 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 47 38, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:37:55+09:00", "last_error_summary": "[h264 @ 0x55a1ab885580] cabac decode of qscale diff failed at 37 41 [h264 @ 0x55a1ab885580] error while decoding MB 37 41, bytestream 34956 [h264 @ 0x55a1ac372d40] error while decoding MB 20 22, bytestream -9 [h264 @ 0x55a1ac2a3ec0] cabac decode of qscale diff failed at 57 13 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 57 13, bytestream 46425 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 28 24, bytestream -9 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 47 38, bytestream -12", "last_exit_at": "2026-07-06T11:37:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.703, "last_started_at": "2026-07-06T11:35:36+09:00", "pid": 2067, "process": {"exists": false, "pid": 2067}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.967, "sample_count": 1318, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:37:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T11:37:55+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T11:37:54+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-06T11:37:54.975799+09:00", "mtime_epoch": 1783305474.9757986, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186793}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:37:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.098, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.929, "pushed": 4.929, "read": 4.895}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.575, "count": 147, "max_ms": 15.0, "p95_ms": 11.59}, "gstreamer_push": {"avg_ms": 0.891, "count": 147, "max_ms": 1.113, "p95_ms": 0.998}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.095, "count": 147, "max_ms": 83.439, "p95_ms": 74.457}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.33, "count": 147, "max_ms": 116.45, "p95_ms": 108.108}, "yolo11_640_fire_11class": {"avg_ms": 58.51, "count": 147, "max_ms": 85.753, "p95_ms": 78.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.54, "count": 147, "max_ms": 111.171, "p95_ms": 99.926}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.291, "count": 147, "max_ms": 250.566, "p95_ms": 235.855}, "read_interval": {"avg_ms": 204.725, "count": 146, "max_ms": 257.13, "p95_ms": 244.15}, "triton_wait": {"avg_ms": 77.714, "count": 147, "max_ms": 116.462, "p95_ms": 108.438}, "visualize": {"avg_ms": 8.536, "count": 147, "max_ms": 11.642, "p95_ms": 10.655}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.824, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 60}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4513.855, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:37:55+09:00", "last_error_summary": "[h264 @ 0x55a1ab885580] cabac decode of qscale diff failed at 37 41 [h264 @ 0x55a1ab885580] error while decoding MB 37 41, bytestream 34956 [h264 @ 0x55a1ac372d40] error while decoding MB 20 22, bytestream -9 [h264 @ 0x55a1ac2a3ec0] cabac decode of qscale diff failed at 57 13 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 57 13, bytestream 46425 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 28 24, bytestream -9 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 47 38, bytestream -12", "last_exit_at": "2026-07-06T11:37:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.708, "last_started_at": "2026-07-06T11:35:36+09:00", "pid": 2067, "process": {"exists": false, "pid": 2067}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.967, "sample_count": 1318, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:37:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T11:37:55+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-06T11:37:54+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-06T11:37:54.975799+09:00", "mtime_epoch": 1783305474.9757986, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186793}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:37:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.098, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.928, "pushed": 4.928, "read": 4.895}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.575, "count": 147, "max_ms": 15.0, "p95_ms": 11.59}, "gstreamer_push": {"avg_ms": 0.891, "count": 147, "max_ms": 1.113, "p95_ms": 0.998}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.095, "count": 147, "max_ms": 83.439, "p95_ms": 74.457}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.33, "count": 147, "max_ms": 116.45, "p95_ms": 108.108}, "yolo11_640_fire_11class": {"avg_ms": 58.51, "count": 147, "max_ms": 85.753, "p95_ms": 78.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.54, "count": 147, "max_ms": 111.171, "p95_ms": 99.926}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.291, "count": 147, "max_ms": 250.566, "p95_ms": 235.855}, "read_interval": {"avg_ms": 204.725, "count": 146, "max_ms": 257.13, "p95_ms": 244.15}, "triton_wait": {"avg_ms": 77.714, "count": 147, "max_ms": 116.462, "p95_ms": 108.438}, "visualize": {"avg_ms": 8.536, "count": 147, "max_ms": 11.642, "p95_ms": 10.655}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.828, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 60}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4616.844, "limit_mb": 8796093022207.996}, "detail": {"pid": 2234, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:37:55+09:00", "last_error_summary": "[h264 @ 0x55a1ab885580] cabac decode of qscale diff failed at 37 41 [h264 @ 0x55a1ab885580] error while decoding MB 37 41, bytestream 34956 [h264 @ 0x55a1ac372d40] error while decoding MB 20 22, bytestream -9 [h264 @ 0x55a1ac2a3ec0] cabac decode of qscale diff failed at 57 13 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 57 13, bytestream 46425 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 28 24, bytestream -9 [h264 @ 0x55a1ac2a3ec0] error while decoding MB 47 38, bytestream -12", "last_exit_at": "2026-07-06T11:37:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:37:57+09:00", "pid": 2234, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.355, "name": "ffmpeg", "pid": 2234, "rss_mb": 136.355, "state": "S (sleeping)", "threads": 106}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.119, "sample_count": 1295, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:37:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.176, "last_frame_pushed_at": "2026-07-06T11:37:55+09:00", "last_frame_read_age_sec": 4.324, "last_frame_read_at": "2026-07-06T11:37:54+09:00", "mjpeg": {"age_sec": 4.164, "exists": true, "mtime": "2026-07-06T11:37:55.123801+09:00", "mtime_epoch": 1783305475.123801, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:37:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.098, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.269, "pushed": 4.269, "read": 4.235}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.46, "count": 127, "max_ms": 15.0, "p95_ms": 11.348}, "gstreamer_push": {"avg_ms": 0.889, "count": 128, "max_ms": 1.113, "p95_ms": 0.998}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.901, "count": 127, "max_ms": 83.439, "p95_ms": 75.026}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.974, "count": 127, "max_ms": 116.45, "p95_ms": 108.108}, "yolo11_640_fire_11class": {"avg_ms": 58.451, "count": 127, "max_ms": 85.753, "p95_ms": 78.371}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.521, "count": 127, "max_ms": 111.171, "p95_ms": 100.049}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.389, "count": 128, "max_ms": 250.566, "p95_ms": 235.855}, "read_interval": {"avg_ms": 203.855, "count": 127, "max_ms": 257.13, "p95_ms": 244.15}, "triton_wait": {"avg_ms": 77.376, "count": 127, "max_ms": 116.462, "p95_ms": 108.438}, "visualize": {"avg_ms": 8.477, "count": 127, "max_ms": 11.642, "p95_ms": 10.962}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 87}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4452.672, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559e26091600] error while decoding MB 62 47, bytestream -25 [in#0/rtsp @ 0x559e2604ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e2604ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:39:00+09:00", "last_error_summary": "[h264 @ 0x559e26091600] error while decoding MB 62 47, bytestream -25 [in#0/rtsp @ 0x559e2604ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e2604ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:21:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 1034.428, "last_started_at": "2026-07-06T11:21:46+09:00", "pid": 1088, "process": {"exists": false, "pid": 1088}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.908, "sample_count": 2774, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:39:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-06T11:39:00+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T11:39:00+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-06T11:39:00.408920+09:00", "mtime_epoch": 1783305540.4089198, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:39:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1315.629, "name": "celery", "pid": 267, "rss_mb": 1058.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 28, "pushed_frames": 28, "read_frames": 29, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.933, "pushed": 0.933, "read": 0.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.664, "count": 28, "max_ms": 4.799, "p95_ms": 4.585}, "gstreamer_push": {"avg_ms": 1.945, "count": 28, "max_ms": 6.5, "p95_ms": 6.013}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.988, "count": 28, "max_ms": 84.862, "p95_ms": 79.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.747, "count": 28, "max_ms": 108.277, "p95_ms": 102.24}, "yolo11_640_fire_11class": {"avg_ms": 60.573, "count": 28, "max_ms": 84.845, "p95_ms": 79.387}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.034, "count": 28, "max_ms": 105.821, "p95_ms": 99.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.277, "count": 28, "max_ms": 120.669, "p95_ms": 113.787}, "read_interval": {"avg_ms": 1034.986, "count": 29, "max_ms": 18135.456, "p95_ms": 9179.58}, "triton_wait": {"avg_ms": 75.759, "count": 28, "max_ms": 108.289, "p95_ms": 102.251}, "visualize": {"avg_ms": 3.681, "count": 28, "max_ms": 4.403, "p95_ms": 4.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 87}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4452.254, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:39:00+09:00", "last_error_summary": "[h264 @ 0x559e26091600] error while decoding MB 62 47, bytestream -25 [in#0/rtsp @ 0x559e2604ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e2604ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:39:00+09:00", "last_error_summary": "[h264 @ 0x559e26091600] error while decoding MB 62 47, bytestream -25 [in#0/rtsp @ 0x559e2604ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e2604ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:39:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 1034.452, "last_started_at": "2026-07-06T11:21:46+09:00", "pid": 1088, "process": {"exists": false, "pid": 1088}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.908, "sample_count": 2774, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10441, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:39:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T11:39:00+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-06T11:39:00+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-06T11:39:00.408920+09:00", "mtime_epoch": 1783305540.4089198, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:39:00+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1315.629, "name": "celery", "pid": 267, "rss_mb": 1058.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.969, "pushed": 0.969, "read": 0.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.808, "count": 29, "max_ms": 7.827, "p95_ms": 4.799}, "gstreamer_push": {"avg_ms": 1.921, "count": 29, "max_ms": 6.5, "p95_ms": 6.013}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.77, "count": 29, "max_ms": 84.862, "p95_ms": 79.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.349, "count": 29, "max_ms": 108.277, "p95_ms": 102.24}, "yolo11_640_fire_11class": {"avg_ms": 60.474, "count": 29, "max_ms": 84.845, "p95_ms": 79.387}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.797, "count": 29, "max_ms": 105.821, "p95_ms": 99.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.051, "count": 29, "max_ms": 120.669, "p95_ms": 113.787}, "read_interval": {"avg_ms": 1068.9, "count": 28, "max_ms": 18135.456, "p95_ms": 9179.58}, "triton_wait": {"avg_ms": 75.362, "count": 29, "max_ms": 108.289, "p95_ms": 102.251}, "visualize": {"avg_ms": 3.715, "count": 29, "max_ms": 4.674, "p95_ms": 4.403}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 87}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4452.117, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:39:00+09:00", "last_error_summary": "[h264 @ 0x559e26091600] error while decoding MB 62 47, bytestream -25 [in#0/rtsp @ 0x559e2604ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e2604ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:39:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 1034.459, "last_started_at": "2026-07-06T11:21:46+09:00", "pid": 1088, "process": {"exists": false, "pid": 1088}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.908, "sample_count": 2774, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10441, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:39:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T11:39:00+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-06T11:39:00+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-06T11:39:00.408920+09:00", "mtime_epoch": 1783305540.4089198, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:39:00+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1315.629, "name": "celery", "pid": 267, "rss_mb": 1058.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.969, "pushed": 0.969, "read": 0.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.808, "count": 29, "max_ms": 7.827, "p95_ms": 4.799}, "gstreamer_push": {"avg_ms": 1.921, "count": 29, "max_ms": 6.5, "p95_ms": 6.013}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.77, "count": 29, "max_ms": 84.862, "p95_ms": 79.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.349, "count": 29, "max_ms": 108.277, "p95_ms": 102.24}, "yolo11_640_fire_11class": {"avg_ms": 60.474, "count": 29, "max_ms": 84.845, "p95_ms": 79.387}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.797, "count": 29, "max_ms": 105.821, "p95_ms": 99.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.051, "count": 29, "max_ms": 120.669, "p95_ms": 113.787}, "read_interval": {"avg_ms": 1068.9, "count": 28, "max_ms": 18135.456, "p95_ms": 9179.58}, "triton_wait": {"avg_ms": 75.362, "count": 29, "max_ms": 108.289, "p95_ms": 102.251}, "visualize": {"avg_ms": 3.715, "count": 29, "max_ms": 4.674, "p95_ms": 4.403}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 87}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4585.598, "limit_mb": 8796093022207.996}, "detail": {"pid": 2341, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:39:00+09:00", "last_error_summary": "[h264 @ 0x559e26091600] error while decoding MB 62 47, bytestream -25 [in#0/rtsp @ 0x559e2604ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e2604ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:39:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:39:02+09:00", "pid": 2341, "process": {"exists": true, "fd_count": 5, "hwm_mb": 157.289, "name": "ffmpeg", "pid": 2341, "rss_mb": 157.125, "state": "R (running)", "threads": 49}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.578, "sample_count": 2734, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10441, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:39:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 4.26, "last_frame_pushed_at": "2026-07-06T11:39:00+09:00", "last_frame_read_age_sec": 4.344, "last_frame_read_at": "2026-07-06T11:39:00+09:00", "mjpeg": {"age_sec": 4.246, "exists": true, "mtime": "2026-07-06T11:39:00.504921+09:00", "mtime_epoch": 1783305540.5049214, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224396}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:39:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1315.629, "name": "celery", "pid": 267, "rss_mb": 1058.285, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.551, "pushed": 0.551, "read": 0.551}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.092, "count": 13, "max_ms": 7.827, "p95_ms": 7.827}, "gstreamer_push": {"avg_ms": 2.259, "count": 13, "max_ms": 6.5, "p95_ms": 6.5}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.498, "count": 13, "max_ms": 64.231, "p95_ms": 64.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.116, "count": 13, "max_ms": 98.778, "p95_ms": 98.778}, "yolo11_640_fire_11class": {"avg_ms": 59.734, "count": 13, "max_ms": 77.748, "p95_ms": 77.748}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.406, "count": 13, "max_ms": 90.59, "p95_ms": 90.59}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.397, "count": 13, "max_ms": 111.795, "p95_ms": 111.795}, "read_interval": {"avg_ms": 2187.8, "count": 13, "max_ms": 18135.456, "p95_ms": 18135.456}, "triton_wait": {"avg_ms": 76.128, "count": 13, "max_ms": 98.792, "p95_ms": 98.792}, "visualize": {"avg_ms": 3.843, "count": 13, "max_ms": 4.674, "p95_ms": 4.674}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.606, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 70}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4523.547, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 12, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562e2c095180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2c095180] error while decoding MB 0 24, bytestream 3553 [h264 @ 0x562e2bc8a9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2bc8a9c0] error while decoding MB 0 35, bytestream 5085 [h264 @ 0x562e2b3f30c0] cabac decode of qscale diff failed at 33 2 [h264 @ 0x562e2b3f30c0] error while decoding MB 33 2, bytestream 57932 [h264 @ 0x562e2bc8a9c0] cabac decode of qscale diff failed at 21 4", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:40:51+09:00", "last_error_summary": "[h264 @ 0x562e2c095180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2c095180] error while decoding MB 0 24, bytestream 3553 [h264 @ 0x562e2bc8a9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2bc8a9c0] error while decoding MB 0 35, bytestream 5085 [h264 @ 0x562e2b3f30c0] cabac decode of qscale diff failed at 33 2 [h264 @ 0x562e2b3f30c0] error while decoding MB 33 2, bytestream 57932 [h264 @ 0x562e2bc8a9c0] cabac decode of qscale diff failed at 21 4", "last_exit_at": "2026-07-06T11:37:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 173.846, "last_started_at": "2026-07-06T11:37:57+09:00", "pid": 2234, "process": {"exists": false, "pid": 2234}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.837, "sample_count": 1728, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8437, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:40:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-06T11:40:51+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T11:40:51+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T11:40:51.090828+09:00", "mtime_epoch": 1783305651.090828, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:40:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 107, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.533, "pushed": 3.533, "read": 3.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.868, "count": 106, "max_ms": 6.088, "p95_ms": 5.508}, "gstreamer_push": {"avg_ms": 1.309, "count": 106, "max_ms": 1.524, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.58, "count": 106, "max_ms": 87.665, "p95_ms": 76.41}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.717, "count": 106, "max_ms": 114.613, "p95_ms": 109.949}, "yolo11_640_fire_11class": {"avg_ms": 61.306, "count": 106, "max_ms": 90.708, "p95_ms": 84.177}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.751, "count": 106, "max_ms": 107.204, "p95_ms": 97.789}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.894, "count": 106, "max_ms": 127.013, "p95_ms": 122.329}, "read_interval": {"avg_ms": 281.05, "count": 107, "max_ms": 19275.081, "p95_ms": 134.223}, "triton_wait": {"avg_ms": 77.352, "count": 106, "max_ms": 114.626, "p95_ms": 109.962}, "visualize": {"avg_ms": 3.674, "count": 106, "max_ms": 4.767, "p95_ms": 4.294}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 70}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4522.199, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:40:51+09:00", "last_error_summary": "[h264 @ 0x562e2c095180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2c095180] error while decoding MB 0 24, bytestream 3553 [h264 @ 0x562e2bc8a9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2bc8a9c0] error while decoding MB 0 35, bytestream 5085 [h264 @ 0x562e2b3f30c0] cabac decode of qscale diff failed at 33 2 [h264 @ 0x562e2b3f30c0] error while decoding MB 33 2, bytestream 57932 [h264 @ 0x562e2bc8a9c0] cabac decode of qscale diff failed at 21 4", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:40:51+09:00", "last_error_summary": "[h264 @ 0x562e2c095180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2c095180] error while decoding MB 0 24, bytestream 3553 [h264 @ 0x562e2bc8a9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2bc8a9c0] error while decoding MB 0 35, bytestream 5085 [h264 @ 0x562e2b3f30c0] cabac decode of qscale diff failed at 33 2 [h264 @ 0x562e2b3f30c0] error while decoding MB 33 2, bytestream 57932 [h264 @ 0x562e2bc8a9c0] cabac decode of qscale diff failed at 21 4", "last_exit_at": "2026-07-06T11:40:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 173.915, "last_started_at": "2026-07-06T11:37:57+09:00", "pid": 2234, "process": {"exists": false, "pid": 2234}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.837, "sample_count": 1729, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8438, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:40:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T11:40:51+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-06T11:40:51+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T11:40:51.090828+09:00", "mtime_epoch": 1783305651.090828, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:40:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.883, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.572, "pushed": 3.572, "read": 3.538}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.867, "count": 107, "max_ms": 6.088, "p95_ms": 5.508}, "gstreamer_push": {"avg_ms": 1.309, "count": 107, "max_ms": 1.524, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.663, "count": 107, "max_ms": 87.665, "p95_ms": 76.41}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.917, "count": 107, "max_ms": 114.613, "p95_ms": 109.949}, "yolo11_640_fire_11class": {"avg_ms": 61.347, "count": 107, "max_ms": 90.708, "p95_ms": 84.177}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.759, "count": 107, "max_ms": 107.204, "p95_ms": 97.789}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.086, "count": 107, "max_ms": 127.013, "p95_ms": 122.329}, "read_interval": {"avg_ms": 282.514, "count": 106, "max_ms": 19275.081, "p95_ms": 134.223}, "triton_wait": {"avg_ms": 77.546, "count": 107, "max_ms": 114.626, "p95_ms": 109.962}, "visualize": {"avg_ms": 3.674, "count": 107, "max_ms": 4.767, "p95_ms": 4.294}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 70}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4522.191, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:40:51+09:00", "last_error_summary": "[h264 @ 0x562e2c095180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2c095180] error while decoding MB 0 24, bytestream 3553 [h264 @ 0x562e2bc8a9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2bc8a9c0] error while decoding MB 0 35, bytestream 5085 [h264 @ 0x562e2b3f30c0] cabac decode of qscale diff failed at 33 2 [h264 @ 0x562e2b3f30c0] error while decoding MB 33 2, bytestream 57932 [h264 @ 0x562e2bc8a9c0] cabac decode of qscale diff failed at 21 4", "last_exit_at": "2026-07-06T11:40:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 173.924, "last_started_at": "2026-07-06T11:37:57+09:00", "pid": 2234, "process": {"exists": false, "pid": 2234}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.837, "sample_count": 1729, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8438, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:40:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T11:40:51+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-06T11:40:51+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T11:40:51.090828+09:00", "mtime_epoch": 1783305651.090828, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:40:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.883, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.571, "pushed": 3.571, "read": 3.537}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.867, "count": 107, "max_ms": 6.088, "p95_ms": 5.508}, "gstreamer_push": {"avg_ms": 1.309, "count": 107, "max_ms": 1.524, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.663, "count": 107, "max_ms": 87.665, "p95_ms": 76.41}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.917, "count": 107, "max_ms": 114.613, "p95_ms": 109.949}, "yolo11_640_fire_11class": {"avg_ms": 61.347, "count": 107, "max_ms": 90.708, "p95_ms": 84.177}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.759, "count": 107, "max_ms": 107.204, "p95_ms": 97.789}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.086, "count": 107, "max_ms": 127.013, "p95_ms": 122.329}, "read_interval": {"avg_ms": 282.514, "count": 106, "max_ms": 19275.081, "p95_ms": 134.223}, "triton_wait": {"avg_ms": 77.546, "count": 107, "max_ms": 114.626, "p95_ms": 109.962}, "visualize": {"avg_ms": 3.674, "count": 107, "max_ms": 4.767, "p95_ms": 4.294}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 70}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4626.992, "limit_mb": 8796093022207.996}, "detail": {"pid": 2493, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:40:51+09:00", "last_error_summary": "[h264 @ 0x562e2c095180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2c095180] error while decoding MB 0 24, bytestream 3553 [h264 @ 0x562e2bc8a9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562e2bc8a9c0] error while decoding MB 0 35, bytestream 5085 [h264 @ 0x562e2b3f30c0] cabac decode of qscale diff failed at 33 2 [h264 @ 0x562e2b3f30c0] error while decoding MB 33 2, bytestream 57932 [h264 @ 0x562e2bc8a9c0] cabac decode of qscale diff failed at 21 4", "last_exit_at": "2026-07-06T11:40:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:40:53+09:00", "pid": 2493, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.02, "name": "ffmpeg", "pid": 2493, "rss_mb": 136.02, "state": "S (sleeping)", "threads": 106}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.994, "sample_count": 1705, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8438, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:40:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.346, "last_frame_pushed_at": "2026-07-06T11:40:51+09:00", "last_frame_read_age_sec": 4.459, "last_frame_read_at": "2026-07-06T11:40:51+09:00", "mjpeg": {"age_sec": 4.324, "exists": true, "mtime": "2026-07-06T11:40:51.214830+09:00", "mtime_epoch": 1783305651.21483, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:40:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1089.883, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.17, "pushed": 2.17, "read": 2.137}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.941, "count": 65, "max_ms": 6.088, "p95_ms": 5.639}, "gstreamer_push": {"avg_ms": 1.313, "count": 65, "max_ms": 1.524, "p95_ms": 1.446}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.18, "count": 65, "max_ms": 76.888, "p95_ms": 67.603}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.577, "count": 65, "max_ms": 111.059, "p95_ms": 108.112}, "yolo11_640_fire_11class": {"avg_ms": 60.635, "count": 65, "max_ms": 86.865, "p95_ms": 83.198}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.638, "count": 65, "max_ms": 102.738, "p95_ms": 97.569}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.744, "count": 65, "max_ms": 126.364, "p95_ms": 120.5}, "read_interval": {"avg_ms": 399.748, "count": 64, "max_ms": 19275.081, "p95_ms": 133.053}, "triton_wait": {"avg_ms": 76.124, "count": 65, "max_ms": 111.072, "p95_ms": 108.124}, "visualize": {"avg_ms": 3.697, "count": 65, "max_ms": 4.767, "p95_ms": 4.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4463.68, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c22ccfb200] error while decoding MB 83 33, bytestream -21 [in#0/rtsp @ 0x55c22cb3ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c22cb3ee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:42:34+09:00", "last_error_summary": "[h264 @ 0x55c22ccfb200] error while decoding MB 83 33, bytestream -21 [in#0/rtsp @ 0x55c22cb3ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c22cb3ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:39:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 212.105, "last_started_at": "2026-07-06T11:39:02+09:00", "pid": 2341, "process": {"exists": false, "pid": 2341}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.052, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12079, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:42:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T11:42:34+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T11:42:34+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-06T11:42:34.792626+09:00", "mtime_epoch": 1783305754.7926261, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 235085}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:42:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1315.629, "name": "celery", "pid": 267, "rss_mb": 1054.199, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.445, "pushed": 2.445, "read": 2.445}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.653, "count": 73, "max_ms": 7.906, "p95_ms": 6.661}, "gstreamer_push": {"avg_ms": 1.363, "count": 73, "max_ms": 6.223, "p95_ms": 3.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.531, "count": 73, "max_ms": 77.114, "p95_ms": 70.647}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.873, "count": 73, "max_ms": 118.454, "p95_ms": 109.052}, "yolo11_640_fire_11class": {"avg_ms": 64.693, "count": 73, "max_ms": 93.25, "p95_ms": 83.641}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.452, "count": 73, "max_ms": 99.323, "p95_ms": 94.941}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.591, "count": 73, "max_ms": 170.88, "p95_ms": 165.525}, "read_interval": {"avg_ms": 410.393, "count": 73, "max_ms": 11082.521, "p95_ms": 178.966}, "triton_wait": {"avg_ms": 82.088, "count": 73, "max_ms": 118.466, "p95_ms": 109.066}, "visualize": {"avg_ms": 4.924, "count": 73, "max_ms": 5.675, "p95_ms": 5.372}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4462.535, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:42:34+09:00", "last_error_summary": "[h264 @ 0x55c22ccfb200] error while decoding MB 83 33, bytestream -21 [in#0/rtsp @ 0x55c22cb3ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c22cb3ee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:42:34+09:00", "last_error_summary": "[h264 @ 0x55c22ccfb200] error while decoding MB 83 33, bytestream -21 [in#0/rtsp @ 0x55c22cb3ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c22cb3ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:42:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 212.174, "last_started_at": "2026-07-06T11:39:02+09:00", "pid": 2341, "process": {"exists": false, "pid": 2341}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.052, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12080, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:42:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T11:42:34+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T11:42:34+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-06T11:42:34.792626+09:00", "mtime_epoch": 1783305754.7926261, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 235085}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:42:34+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1315.629, "name": "celery", "pid": 267, "rss_mb": 1054.203, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.473, "pushed": 2.473, "read": 2.439}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.744, "count": 74, "max_ms": 12.397, "p95_ms": 6.806}, "gstreamer_push": {"avg_ms": 1.361, "count": 74, "max_ms": 6.223, "p95_ms": 3.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.348, "count": 74, "max_ms": 77.114, "p95_ms": 70.647}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.527, "count": 74, "max_ms": 118.454, "p95_ms": 109.052}, "yolo11_640_fire_11class": {"avg_ms": 64.519, "count": 74, "max_ms": 93.25, "p95_ms": 83.641}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.233, "count": 74, "max_ms": 99.323, "p95_ms": 94.941}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.515, "count": 74, "max_ms": 170.88, "p95_ms": 165.525}, "read_interval": {"avg_ms": 410.393, "count": 73, "max_ms": 11082.521, "p95_ms": 178.966}, "triton_wait": {"avg_ms": 81.739, "count": 74, "max_ms": 118.466, "p95_ms": 109.066}, "visualize": {"avg_ms": 4.964, "count": 74, "max_ms": 7.92, "p95_ms": 5.4}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4462.527, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:42:34+09:00", "last_error_summary": "[h264 @ 0x55c22ccfb200] error while decoding MB 83 33, bytestream -21 [in#0/rtsp @ 0x55c22cb3ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c22cb3ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:42:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 212.18, "last_started_at": "2026-07-06T11:39:02+09:00", "pid": 2341, "process": {"exists": false, "pid": 2341}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.052, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12080, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:42:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T11:42:34+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-06T11:42:34+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-06T11:42:34.792626+09:00", "mtime_epoch": 1783305754.7926261, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 235085}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:42:34+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1315.629, "name": "celery", "pid": 267, "rss_mb": 1054.203, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.472, "pushed": 2.472, "read": 2.439}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.744, "count": 74, "max_ms": 12.397, "p95_ms": 6.806}, "gstreamer_push": {"avg_ms": 1.361, "count": 74, "max_ms": 6.223, "p95_ms": 3.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.348, "count": 74, "max_ms": 77.114, "p95_ms": 70.647}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.527, "count": 74, "max_ms": 118.454, "p95_ms": 109.052}, "yolo11_640_fire_11class": {"avg_ms": 64.519, "count": 74, "max_ms": 93.25, "p95_ms": 83.641}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.233, "count": 74, "max_ms": 99.323, "p95_ms": 94.941}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.515, "count": 74, "max_ms": 170.88, "p95_ms": 165.525}, "read_interval": {"avg_ms": 410.393, "count": 73, "max_ms": 11082.521, "p95_ms": 178.966}, "triton_wait": {"avg_ms": 81.739, "count": 74, "max_ms": 118.466, "p95_ms": 109.066}, "visualize": {"avg_ms": 4.964, "count": 74, "max_ms": 7.92, "p95_ms": 5.4}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4607.242, "limit_mb": 8796093022207.996}, "detail": {"pid": 2600, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:42:34+09:00", "last_error_summary": "[h264 @ 0x55c22ccfb200] error while decoding MB 83 33, bytestream -21 [in#0/rtsp @ 0x55c22cb3ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c22cb3ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:42:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:42:37+09:00", "pid": 2600, "process": {"exists": true, "fd_count": 5, "hwm_mb": 167.102, "name": "ffmpeg", "pid": 2600, "rss_mb": 167.102, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.286, "sample_count": 2178, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12080, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:42:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 4.89, "last_frame_pushed_at": "2026-07-06T11:42:34+09:00", "last_frame_read_age_sec": 5.016, "last_frame_read_at": "2026-07-06T11:42:34+09:00", "mjpeg": {"age_sec": 4.88, "exists": true, "mtime": "2026-07-06T11:42:34.928629+09:00", "mtime_epoch": 1783305754.9286287, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234967}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:42:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1315.629, "name": "celery", "pid": 267, "rss_mb": 1054.203, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.337, "pushed": 1.337, "read": 1.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.618, "count": 40, "max_ms": 12.397, "p95_ms": 7.034}, "gstreamer_push": {"avg_ms": 1.571, "count": 40, "max_ms": 6.223, "p95_ms": 3.965}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.636, "count": 40, "max_ms": 75.876, "p95_ms": 70.017}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.61, "count": 40, "max_ms": 116.267, "p95_ms": 104.977}, "yolo11_640_fire_11class": {"avg_ms": 64.466, "count": 40, "max_ms": 93.25, "p95_ms": 84.289}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.169, "count": 40, "max_ms": 99.323, "p95_ms": 93.456}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.712, "count": 40, "max_ms": 169.171, "p95_ms": 158.388}, "read_interval": {"avg_ms": 642.557, "count": 39, "max_ms": 11082.521, "p95_ms": 8867.575}, "triton_wait": {"avg_ms": 81.992, "count": 40, "max_ms": 116.281, "p95_ms": 104.989}, "visualize": {"avg_ms": 4.85, "count": 40, "max_ms": 7.92, "p95_ms": 5.631}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 90}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4619.23, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563da6a0d480] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563da6a0d480] error while decoding MB 0 27, bytestream 849 [h264 @ 0x563da5dfbc80] error while decoding MB 21 32, bytestream -5 [h264 @ 0x563da5f1a640] cabac decode of qscale diff failed at 26 41 [h264 @ 0x563da5f1a640] error while decoding MB 26 41, bytestream -6 [in#0/rtsp @ 0x563da5ddbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563da5ddbe00] Error retrieving a packet from demuxer: Conne", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:46:14+09:00", "last_error_summary": "[h264 @ 0x563da6a0d480] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563da6a0d480] error while decoding MB 0 27, bytestream 849 [h264 @ 0x563da5dfbc80] error while decoding MB 21 32, bytestream -5 [h264 @ 0x563da5f1a640] cabac decode of qscale diff failed at 26 41 [h264 @ 0x563da5f1a640] error while decoding MB 26 41, bytestream -6 [in#0/rtsp @ 0x563da5ddbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563da5ddbe00] Error retrieving a packet from demuxer: Conne", "last_exit_at": "2026-07-06T11:40:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 321.192, "last_started_at": "2026-07-06T11:40:53+09:00", "pid": 2493, "process": {"exists": false, "pid": 2493}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.529, "sample_count": 2196, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10816, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T11:46:14+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T11:46:14+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-06T11:46:14.684465+09:00", "mtime_epoch": 1783305974.6844654, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181994}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:46:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1090.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 74, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.47, "pushed": 2.47, "read": 2.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.38, "count": 73, "max_ms": 9.95, "p95_ms": 8.087}, "gstreamer_push": {"avg_ms": 1.152, "count": 74, "max_ms": 1.374, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.07, "count": 73, "max_ms": 80.218, "p95_ms": 71.911}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 91.107, "count": 73, "max_ms": 126.144, "p95_ms": 112.786}, "yolo11_640_fire_11class": {"avg_ms": 64.575, "count": 73, "max_ms": 87.689, "p95_ms": 83.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.814, "count": 73, "max_ms": 106.141, "p95_ms": 100.597}}, "truncated_count": 0}, "processing_total": {"avg_ms": 144.769, "count": 74, "max_ms": 178.207, "p95_ms": 168.954}, "read_interval": {"avg_ms": 405.788, "count": 74, "max_ms": 18712.162, "p95_ms": 179.484}, "triton_wait": {"avg_ms": 91.131, "count": 73, "max_ms": 126.158, "p95_ms": 112.799}, "visualize": {"avg_ms": 5.161, "count": 73, "max_ms": 7.226, "p95_ms": 5.905}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 90}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4618.438, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:46:14+09:00", "last_error_summary": "[h264 @ 0x563da6a0d480] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563da6a0d480] error while decoding MB 0 27, bytestream 849 [h264 @ 0x563da5dfbc80] error while decoding MB 21 32, bytestream -5 [h264 @ 0x563da5f1a640] cabac decode of qscale diff failed at 26 41 [h264 @ 0x563da5f1a640] error while decoding MB 26 41, bytestream -6 [in#0/rtsp @ 0x563da5ddbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563da5ddbe00] Error retrieving a packet from demuxer: Conne", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:46:14+09:00", "last_error_summary": "[h264 @ 0x563da6a0d480] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563da6a0d480] error while decoding MB 0 27, bytestream 849 [h264 @ 0x563da5dfbc80] error while decoding MB 21 32, bytestream -5 [h264 @ 0x563da5f1a640] cabac decode of qscale diff failed at 26 41 [h264 @ 0x563da5f1a640] error while decoding MB 26 41, bytestream -6 [in#0/rtsp @ 0x563da5ddbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563da5ddbe00] Error retrieving a packet from demuxer: Conne", "last_exit_at": "2026-07-06T11:46:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 321.264, "last_started_at": "2026-07-06T11:40:53+09:00", "pid": 2493, "process": {"exists": false, "pid": 2493}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.529, "sample_count": 2196, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10817, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T11:46:14+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-06T11:46:14+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T11:46:14.684465+09:00", "mtime_epoch": 1783305974.6844654, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181994}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:46:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1090.914, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.475, "pushed": 2.475, "read": 2.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.385, "count": 74, "max_ms": 9.95, "p95_ms": 8.087}, "gstreamer_push": {"avg_ms": 1.153, "count": 74, "max_ms": 1.374, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.916, "count": 74, "max_ms": 80.218, "p95_ms": 71.911}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.762, "count": 74, "max_ms": 126.144, "p95_ms": 112.786}, "yolo11_640_fire_11class": {"avg_ms": 64.497, "count": 74, "max_ms": 87.689, "p95_ms": 83.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.631, "count": 74, "max_ms": 106.141, "p95_ms": 100.597}}, "truncated_count": 0}, "processing_total": {"avg_ms": 144.802, "count": 74, "max_ms": 178.207, "p95_ms": 168.954}, "read_interval": {"avg_ms": 409.571, "count": 73, "max_ms": 18712.162, "p95_ms": 179.484}, "triton_wait": {"avg_ms": 90.786, "count": 74, "max_ms": 126.158, "p95_ms": 112.799}, "visualize": {"avg_ms": 5.161, "count": 74, "max_ms": 7.226, "p95_ms": 5.905}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 90}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4618.43, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:46:14+09:00", "last_error_summary": "[h264 @ 0x563da6a0d480] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563da6a0d480] error while decoding MB 0 27, bytestream 849 [h264 @ 0x563da5dfbc80] error while decoding MB 21 32, bytestream -5 [h264 @ 0x563da5f1a640] cabac decode of qscale diff failed at 26 41 [h264 @ 0x563da5f1a640] error while decoding MB 26 41, bytestream -6 [in#0/rtsp @ 0x563da5ddbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563da5ddbe00] Error retrieving a packet from demuxer: Conne", "last_exit_at": "2026-07-06T11:46:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 321.269, "last_started_at": "2026-07-06T11:40:53+09:00", "pid": 2493, "process": {"exists": false, "pid": 2493}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.529, "sample_count": 2196, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10817, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T11:46:14+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T11:46:14+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-06T11:46:14.684465+09:00", "mtime_epoch": 1783305974.6844654, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181994}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:46:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1090.914, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.474, "pushed": 2.474, "read": 2.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.385, "count": 74, "max_ms": 9.95, "p95_ms": 8.087}, "gstreamer_push": {"avg_ms": 1.153, "count": 74, "max_ms": 1.374, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.916, "count": 74, "max_ms": 80.218, "p95_ms": 71.911}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.762, "count": 74, "max_ms": 126.144, "p95_ms": 112.786}, "yolo11_640_fire_11class": {"avg_ms": 64.497, "count": 74, "max_ms": 87.689, "p95_ms": 83.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.631, "count": 74, "max_ms": 106.141, "p95_ms": 100.597}}, "truncated_count": 0}, "processing_total": {"avg_ms": 144.802, "count": 74, "max_ms": 178.207, "p95_ms": 168.954}, "read_interval": {"avg_ms": 409.571, "count": 73, "max_ms": 18712.162, "p95_ms": 179.484}, "triton_wait": {"avg_ms": 90.786, "count": 74, "max_ms": 126.158, "p95_ms": 112.799}, "visualize": {"avg_ms": 5.161, "count": 74, "max_ms": 7.226, "p95_ms": 5.905}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 90}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4727.332, "limit_mb": 8796093022207.996}, "detail": {"pid": 2752, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:46:14+09:00", "last_error_summary": "[h264 @ 0x563da6a0d480] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563da6a0d480] error while decoding MB 0 27, bytestream 849 [h264 @ 0x563da5dfbc80] error while decoding MB 21 32, bytestream -5 [h264 @ 0x563da5f1a640] cabac decode of qscale diff failed at 26 41 [h264 @ 0x563da5f1a640] error while decoding MB 26 41, bytestream -6 [in#0/rtsp @ 0x563da5ddbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563da5ddbe00] Error retrieving a packet from demuxer: Conne", "last_exit_at": "2026-07-06T11:46:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:46:17+09:00", "pid": 2752, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.184, "name": "ffmpeg", "pid": 2752, "rss_mb": 139.184, "state": "S (sleeping)", "threads": 106}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.736, "sample_count": 2157, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10817, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.383, "last_frame_pushed_at": "2026-07-06T11:46:14+09:00", "last_frame_read_age_sec": 4.505, "last_frame_read_at": "2026-07-06T11:46:14+09:00", "mjpeg": {"age_sec": 4.368, "exists": true, "mtime": "2026-07-06T11:46:14.812468+09:00", "mtime_epoch": 1783305974.8124676, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182037}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:46:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1090.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.507, "pushed": 1.507, "read": 1.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.484, "count": 45, "max_ms": 9.95, "p95_ms": 8.164}, "gstreamer_push": {"avg_ms": 1.15, "count": 45, "max_ms": 1.363, "p95_ms": 1.242}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.755, "count": 45, "max_ms": 78.628, "p95_ms": 71.46}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 93.689, "count": 45, "max_ms": 114.078, "p95_ms": 112.305}, "yolo11_640_fire_11class": {"avg_ms": 65.594, "count": 45, "max_ms": 87.689, "p95_ms": 83.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.508, "count": 45, "max_ms": 106.141, "p95_ms": 101.359}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.465, "count": 45, "max_ms": 176.666, "p95_ms": 168.954}, "read_interval": {"avg_ms": 580.003, "count": 44, "max_ms": 18712.162, "p95_ms": 179.484}, "triton_wait": {"avg_ms": 93.703, "count": 45, "max_ms": 114.103, "p95_ms": 112.319}, "visualize": {"avg_ms": 5.131, "count": 45, "max_ms": 7.226, "p95_ms": 6.046}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.863, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5039.625, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:49:53+09:00", "last_error_summary": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "last_exit_at": "2026-07-06T11:46:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 216.742, "last_started_at": "2026-07-06T11:46:17+09:00", "pid": 2752, "process": {"exists": false, "pid": 2752}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.345, "sample_count": 1765, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12160, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:49:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.052, "last_frame_pushed_at": "2026-07-06T11:49:53+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-06T11:49:53+09:00", "mjpeg": {"age_sec": 0.033, "exists": true, "mtime": "2026-07-06T11:49:53.888317+09:00", "mtime_epoch": 1783306193.8883173, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192451}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:49:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1091.43, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 76, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.538, "pushed": 2.538, "read": 2.538}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.339, "count": 76, "max_ms": 9.654, "p95_ms": 7.707}, "gstreamer_push": {"avg_ms": 1.17, "count": 76, "max_ms": 1.641, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.417, "count": 76, "max_ms": 89.076, "p95_ms": 78.119}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.279, "count": 76, "max_ms": 115.152, "p95_ms": 109.852}, "yolo11_640_fire_11class": {"avg_ms": 63.727, "count": 76, "max_ms": 88.228, "p95_ms": 82.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.775, "count": 76, "max_ms": 107.669, "p95_ms": 98.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.985, "count": 76, "max_ms": 174.906, "p95_ms": 162.9}, "read_interval": {"avg_ms": 395.27, "count": 76, "max_ms": 18947.395, "p95_ms": 172.927}, "triton_wait": {"avg_ms": 85.366, "count": 76, "max_ms": 115.165, "p95_ms": 109.865}, "visualize": {"avg_ms": 5.181, "count": 76, "max_ms": 7.262, "p95_ms": 5.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5038.711, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:49:53+09:00", "last_error_summary": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:49:53+09:00", "last_error_summary": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "last_exit_at": "2026-07-06T11:49:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 216.81, "last_started_at": "2026-07-06T11:46:17+09:00", "pid": 2752, "process": {"exists": false, "pid": 2752}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.345, "sample_count": 1765, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:49:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T11:49:53+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-06T11:49:53+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-06T11:49:53.888317+09:00", "mtime_epoch": 1783306193.8883173, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192451}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:49:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1091.43, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 75, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.546, "pushed": 2.546, "read": 2.512}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.291, "count": 76, "max_ms": 9.46, "p95_ms": 7.027}, "gstreamer_push": {"avg_ms": 1.169, "count": 76, "max_ms": 1.641, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.405, "count": 76, "max_ms": 89.076, "p95_ms": 78.119}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.932, "count": 76, "max_ms": 115.152, "p95_ms": 109.852}, "yolo11_640_fire_11class": {"avg_ms": 63.66, "count": 76, "max_ms": 88.228, "p95_ms": 82.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.717, "count": 76, "max_ms": 107.669, "p95_ms": 98.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.592, "count": 76, "max_ms": 174.906, "p95_ms": 162.9}, "read_interval": {"avg_ms": 398.58, "count": 75, "max_ms": 18947.395, "p95_ms": 172.927}, "triton_wait": {"avg_ms": 85.019, "count": 76, "max_ms": 115.165, "p95_ms": 109.865}, "visualize": {"avg_ms": 5.178, "count": 76, "max_ms": 7.262, "p95_ms": 5.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5038.699, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:49:53+09:00", "last_error_summary": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "last_exit_at": "2026-07-06T11:49:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 216.815, "last_started_at": "2026-07-06T11:46:17+09:00", "pid": 2752, "process": {"exists": false, "pid": 2752}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.345, "sample_count": 1765, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:49:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T11:49:53+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T11:49:53+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T11:49:53.888317+09:00", "mtime_epoch": 1783306193.8883173, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192451}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:49:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1091.43, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 75, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.545, "pushed": 2.545, "read": 2.512}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.291, "count": 76, "max_ms": 9.46, "p95_ms": 7.027}, "gstreamer_push": {"avg_ms": 1.169, "count": 76, "max_ms": 1.641, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.405, "count": 76, "max_ms": 89.076, "p95_ms": 78.119}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.932, "count": 76, "max_ms": 115.152, "p95_ms": 109.852}, "yolo11_640_fire_11class": {"avg_ms": 63.66, "count": 76, "max_ms": 88.228, "p95_ms": 82.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.717, "count": 76, "max_ms": 107.669, "p95_ms": 98.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.592, "count": 76, "max_ms": 174.906, "p95_ms": 162.9}, "read_interval": {"avg_ms": 398.58, "count": 75, "max_ms": 18947.395, "p95_ms": 172.927}, "triton_wait": {"avg_ms": 85.019, "count": 76, "max_ms": 115.165, "p95_ms": 109.865}, "visualize": {"avg_ms": 5.178, "count": 76, "max_ms": 7.262, "p95_ms": 5.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4997.746, "limit_mb": 8796093022207.996}, "detail": {"pid": 2919, "restart_count": 10, "start_count": 11}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:49:53+09:00", "last_error_summary": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "last_exit_at": "2026-07-06T11:49:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:49:56+09:00", "pid": 2919, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.129, "name": "ffmpeg", "pid": 2919, "rss_mb": 139.129, "state": "S (sleeping)", "threads": 106}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.114, "sample_count": 1733, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:49:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.541, "last_frame_pushed_at": "2026-07-06T11:49:53+09:00", "last_frame_read_age_sec": 4.652, "last_frame_read_at": "2026-07-06T11:49:53+09:00", "mjpeg": {"age_sec": 4.527, "exists": true, "mtime": "2026-07-06T11:49:54.000319+09:00", "mtime_epoch": 1783306194.0003192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192366}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:49:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 1091.43, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 47, "pushed_frames": 47, "read_frames": 46, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.571, "pushed": 1.571, "read": 1.538}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.327, "count": 47, "max_ms": 9.46, "p95_ms": 7.707}, "gstreamer_push": {"avg_ms": 1.169, "count": 47, "max_ms": 1.641, "p95_ms": 1.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.682, "count": 47, "max_ms": 78.967, "p95_ms": 75.355}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.946, "count": 47, "max_ms": 115.152, "p95_ms": 105.633}, "yolo11_640_fire_11class": {"avg_ms": 60.323, "count": 47, "max_ms": 82.868, "p95_ms": 79.437}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.518, "count": 47, "max_ms": 101.109, "p95_ms": 98.091}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.548, "count": 47, "max_ms": 174.906, "p95_ms": 159.401}, "read_interval": {"avg_ms": 551.75, "count": 46, "max_ms": 18947.395, "p95_ms": 169.121}, "triton_wait": {"avg_ms": 79.975, "count": 47, "max_ms": 115.165, "p95_ms": 105.644}, "visualize": {"avg_ms": 5.204, "count": 47, "max_ms": 7.262, "p95_ms": 5.442}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 106}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4531.188, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:49:53+09:00", "last_error_summary": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:49:53+09:00", "last_error_summary": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "last_exit_at": "2026-07-06T11:52:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.227, "last_started_at": "2026-07-06T11:49:56+09:00", "pid": 2919, "process": {"exists": false, "pid": 2919}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.62, "sample_count": 1587, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12776, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:52:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T11:52:13+09:00", "last_frame_read_age_sec": 0.246, "last_frame_read_at": "2026-07-06T11:52:13+09:00", "mjpeg": {"age_sec": 0.234, "exists": true, "mtime": "2026-07-06T11:52:13.518780+09:00", "mtime_epoch": 1783306333.5187798, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190752}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:52:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 967.219, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.933, "pushed": 3.933, "read": 3.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.922, "count": 117, "max_ms": 17.61, "p95_ms": 13.183}, "gstreamer_push": {"avg_ms": 0.856, "count": 118, "max_ms": 1.257, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.451, "count": 117, "max_ms": 84.18, "p95_ms": 76.547}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.368, "count": 117, "max_ms": 120.696, "p95_ms": 113.109}, "yolo11_640_fire_11class": {"avg_ms": 59.833, "count": 117, "max_ms": 87.634, "p95_ms": 81.659}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.019, "count": 117, "max_ms": 113.803, "p95_ms": 108.252}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.228, "count": 118, "max_ms": 301.992, "p95_ms": 279.766}, "read_interval": {"avg_ms": 256.404, "count": 117, "max_ms": 511.243, "p95_ms": 287.701}, "triton_wait": {"avg_ms": 82.561, "count": 117, "max_ms": 120.707, "p95_ms": 113.347}, "visualize": {"avg_ms": 10.718, "count": 117, "max_ms": 12.504, "p95_ms": 11.775}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 106}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4531.191, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:49:53+09:00", "last_error_summary": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "last_exit_at": "2026-07-06T11:52:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.23, "last_started_at": "2026-07-06T11:49:56+09:00", "pid": 2919, "process": {"exists": false, "pid": 2919}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.62, "sample_count": 1587, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12776, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:52:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T11:52:13+09:00", "last_frame_read_age_sec": 0.249, "last_frame_read_at": "2026-07-06T11:52:13+09:00", "mjpeg": {"age_sec": 0.237, "exists": true, "mtime": "2026-07-06T11:52:13.518780+09:00", "mtime_epoch": 1783306333.5187798, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190752}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:52:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 967.219, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.9, "pushed": 3.9, "read": 3.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.922, "count": 117, "max_ms": 17.61, "p95_ms": 13.183}, "gstreamer_push": {"avg_ms": 0.856, "count": 117, "max_ms": 1.257, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.451, "count": 117, "max_ms": 84.18, "p95_ms": 76.547}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.368, "count": 117, "max_ms": 120.696, "p95_ms": 113.109}, "yolo11_640_fire_11class": {"avg_ms": 59.833, "count": 117, "max_ms": 87.634, "p95_ms": 81.659}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.019, "count": 117, "max_ms": 113.803, "p95_ms": 108.252}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.245, "count": 117, "max_ms": 301.992, "p95_ms": 279.766}, "read_interval": {"avg_ms": 256.404, "count": 117, "max_ms": 511.243, "p95_ms": 287.701}, "triton_wait": {"avg_ms": 82.561, "count": 117, "max_ms": 120.707, "p95_ms": 113.347}, "visualize": {"avg_ms": 10.718, "count": 117, "max_ms": 12.504, "p95_ms": 11.775}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 106}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4647.691, "limit_mb": 8796093022207.996}, "detail": {"pid": 3026, "restart_count": 11, "start_count": 12}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:49:53+09:00", "last_error_summary": "[h264 @ 0x55856bf41cc0] reference picture missing during reorder [h264 @ 0x55856bf41cc0] Missing reference picture, default is 65537 [h264 @ 0x55856c030180] mmco: unref short failure [h264 @ 0x55856c030180] illegal short term buffer state detected [h264 @ 0x55856cbaae00] left block unavailable for requested intra mode [h264 @ 0x55856cbaae00] error while decoding MB 0 35, bytestream 6795 [h264 @ 0x55856c86f4c0] error while decoding MB 28 41, bytestream -5 [in#0/rtsp @ 0x55856beb9e00] Error during", "last_exit_at": "2026-07-06T11:52:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:52:16+09:00", "pid": 3026, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.992, "name": "ffmpeg", "pid": 3026, "rss_mb": 138.992, "state": "S (sleeping)", "threads": 106}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 263.072, "sample_count": 1558, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12776, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:52:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.353, "last_frame_pushed_at": "2026-07-06T11:52:13+09:00", "last_frame_read_age_sec": 4.594, "last_frame_read_at": "2026-07-06T11:52:13+09:00", "mjpeg": {"age_sec": 4.334, "exists": true, "mtime": "2026-07-06T11:52:13.766784+09:00", "mtime_epoch": 1783306333.7667842, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:52:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1146.734, "name": "celery", "pid": 237, "rss_mb": 967.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.374, "pushed": 3.374, "read": 3.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.887, "count": 100, "max_ms": 17.61, "p95_ms": 13.148}, "gstreamer_push": {"avg_ms": 0.855, "count": 101, "max_ms": 1.257, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.846, "count": 100, "max_ms": 82.165, "p95_ms": 73.906}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.216, "count": 100, "max_ms": 115.223, "p95_ms": 111.493}, "yolo11_640_fire_11class": {"avg_ms": 59.22, "count": 100, "max_ms": 87.634, "p95_ms": 81.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.304, "count": 100, "max_ms": 113.803, "p95_ms": 107.687}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.601, "count": 101, "max_ms": 289.844, "p95_ms": 274.718}, "read_interval": {"avg_ms": 256.141, "count": 100, "max_ms": 511.243, "p95_ms": 284.889}, "triton_wait": {"avg_ms": 81.415, "count": 100, "max_ms": 115.236, "p95_ms": 111.507}, "visualize": {"avg_ms": 10.769, "count": 100, "max_ms": 12.504, "p95_ms": 11.775}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 109}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4986.387, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5651db6fe840] error while decoding MB 60 35, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:54:32+09:00", "last_error_summary": "[h264 @ 0x5651db6fe840] error while decoding MB 60 35, bytestream -5", "last_exit_at": "2026-07-06T11:52:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.57, "last_started_at": "2026-07-06T11:52:16+09:00", "pid": 3026, "process": {"exists": false, "pid": 3026}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 491.857, "sample_count": 947, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13089, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:54:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T11:54:32+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T11:54:32+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T11:54:32.625238+09:00", "mtime_epoch": 1783306472.625238, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:54:32+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1347.543, "name": "celery", "pid": 237, "rss_mb": 1273.988, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 21.853, "recorders": [{"age_sec": 21.853, "frame_count": 91, "is_recording": true, "is_stopping": false, "pid": 3253, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.227, "name": "ffmpeg", "pid": 3253, "rss_mb": 186.227, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 9, "returncode": null, "running": true, "started_at": "2026-07-06T11:54:10+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_115410_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [3253], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 59, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.984, "pushed": 1.984, "read": 1.984}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.944, "count": 59, "max_ms": 38.077, "p95_ms": 32.624}, "gstreamer_push": {"avg_ms": 0.888, "count": 59, "max_ms": 1.435, "p95_ms": 1.142}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.482, "count": 59, "max_ms": 89.992, "p95_ms": 78.143}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.446, "count": 59, "max_ms": 123.647, "p95_ms": 119.855}, "yolo11_640_fire_11class": {"avg_ms": 56.433, "count": 59, "max_ms": 83.682, "p95_ms": 79.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.292, "count": 59, "max_ms": 126.329, "p95_ms": 122.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 502.426, "count": 59, "max_ms": 891.41, "p95_ms": 749.482}, "read_interval": {"avg_ms": 509.711, "count": 59, "max_ms": 897.433, "p95_ms": 757.056}, "triton_wait": {"avg_ms": 90.883, "count": 59, "max_ms": 126.342, "p95_ms": 122.381}, "visualize": {"avg_ms": 19.016, "count": 59, "max_ms": 29.075, "p95_ms": 27.476}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.732, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 109}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4987.527, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:54:32+09:00", "last_error_summary": "[h264 @ 0x5651db6fe840] error while decoding MB 60 35, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:54:32+09:00", "last_error_summary": "[h264 @ 0x5651db6fe840] error while decoding MB 60 35, bytestream -5", "last_exit_at": "2026-07-06T11:54:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.032, "last_started_at": "2026-07-06T11:52:16+09:00", "pid": 3026, "process": {"exists": false, "pid": 3026}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 492.918, "sample_count": 944, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:54:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T11:54:33+09:00", "last_frame_read_age_sec": 0.517, "last_frame_read_at": "2026-07-06T11:54:32+09:00", "mjpeg": {"age_sec": 0.506, "exists": true, "mtime": "2026-07-06T11:54:32.625238+09:00", "mtime_epoch": 1783306472.625238, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:54:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1347.543, "name": "celery", "pid": 237, "rss_mb": 1273.988, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 22.314, "recorders": [{"age_sec": 22.314, "frame_count": 92, "is_recording": true, "is_stopping": false, "pid": 3253, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.227, "name": "ffmpeg", "pid": 3253, "rss_mb": 186.227, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 8, "returncode": null, "running": true, "started_at": "2026-07-06T11:54:10+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_115410_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [3253], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.983, "pushed": 1.983, "read": 1.949}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.075, "count": 58, "max_ms": 38.077, "p95_ms": 32.624}, "gstreamer_push": {"avg_ms": 0.887, "count": 59, "max_ms": 1.435, "p95_ms": 1.142}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.272, "count": 58, "max_ms": 89.992, "p95_ms": 78.143}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.36, "count": 58, "max_ms": 123.647, "p95_ms": 119.855}, "yolo11_640_fire_11class": {"avg_ms": 56.442, "count": 58, "max_ms": 83.682, "p95_ms": 79.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.337, "count": 58, "max_ms": 126.329, "p95_ms": 122.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 504.365, "count": 59, "max_ms": 891.41, "p95_ms": 749.482}, "read_interval": {"avg_ms": 511.557, "count": 58, "max_ms": 897.433, "p95_ms": 757.056}, "triton_wait": {"avg_ms": 90.805, "count": 58, "max_ms": 126.342, "p95_ms": 122.381}, "visualize": {"avg_ms": 19.096, "count": 58, "max_ms": 29.075, "p95_ms": 27.476}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.753, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 109}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4987.586, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:54:32+09:00", "last_error_summary": "[h264 @ 0x5651db6fe840] error while decoding MB 60 35, bytestream -5", "last_exit_at": "2026-07-06T11:54:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.035, "last_started_at": "2026-07-06T11:52:16+09:00", "pid": 3026, "process": {"exists": false, "pid": 3026}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 492.918, "sample_count": 944, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:54:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T11:54:33+09:00", "last_frame_read_age_sec": 0.52, "last_frame_read_at": "2026-07-06T11:54:32+09:00", "mjpeg": {"age_sec": 0.51, "exists": true, "mtime": "2026-07-06T11:54:32.625238+09:00", "mtime_epoch": 1783306472.625238, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:54:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1347.543, "name": "celery", "pid": 237, "rss_mb": 1273.988, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 22.317, "recorders": [{"age_sec": 22.317, "frame_count": 92, "is_recording": true, "is_stopping": false, "pid": 3253, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.227, "name": "ffmpeg", "pid": 3253, "rss_mb": 186.227, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 8, "returncode": null, "running": true, "started_at": "2026-07-06T11:54:10+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_115410_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [3253], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.983, "pushed": 1.983, "read": 1.949}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.075, "count": 58, "max_ms": 38.077, "p95_ms": 32.624}, "gstreamer_push": {"avg_ms": 0.887, "count": 59, "max_ms": 1.435, "p95_ms": 1.142}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.272, "count": 58, "max_ms": 89.992, "p95_ms": 78.143}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.36, "count": 58, "max_ms": 123.647, "p95_ms": 119.855}, "yolo11_640_fire_11class": {"avg_ms": 56.442, "count": 58, "max_ms": 83.682, "p95_ms": 79.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.337, "count": 58, "max_ms": 126.329, "p95_ms": 122.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 504.365, "count": 59, "max_ms": 891.41, "p95_ms": 749.482}, "read_interval": {"avg_ms": 511.557, "count": 58, "max_ms": 897.433, "p95_ms": 757.056}, "triton_wait": {"avg_ms": 90.805, "count": 58, "max_ms": 126.342, "p95_ms": 122.381}, "visualize": {"avg_ms": 19.096, "count": 58, "max_ms": 29.075, "p95_ms": 27.476}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.756, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 109}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5104.645, "limit_mb": 8796093022207.996}, "detail": {"pid": 3313, "restart_count": 12, "start_count": 13}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:54:32+09:00", "last_error_summary": "[h264 @ 0x5651db6fe840] error while decoding MB 60 35, bytestream -5", "last_exit_at": "2026-07-06T11:54:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:54:35+09:00", "pid": 3313, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.117, "name": "ffmpeg", "pid": 3313, "rss_mb": 139.117, "state": "S (sleeping)", "threads": 106}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 492.918, "sample_count": 940, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:54:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.597, "last_frame_pushed_at": "2026-07-06T11:54:33+09:00", "last_frame_read_age_sec": 5.109, "last_frame_read_at": "2026-07-06T11:54:32+09:00", "mjpeg": {"age_sec": 4.579, "exists": true, "mtime": "2026-07-06T11:54:33.145247+09:00", "mtime_epoch": 1783306473.145247, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:54:37+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1347.543, "name": "celery", "pid": 237, "rss_mb": 1273.988, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 26.907, "recorders": [{"age_sec": 26.907, "frame_count": 92, "is_recording": true, "is_stopping": false, "pid": 3253, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.227, "name": "ffmpeg", "pid": 3253, "rss_mb": 186.227, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 8, "returncode": null, "running": true, "started_at": "2026-07-06T11:54:10+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_115410_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [3253], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.636, "pushed": 1.636, "read": 1.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.43, "count": 48, "max_ms": 38.077, "p95_ms": 32.624}, "gstreamer_push": {"avg_ms": 0.891, "count": 49, "max_ms": 1.435, "p95_ms": 1.142}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.679, "count": 48, "max_ms": 79.724, "p95_ms": 62.068}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.875, "count": 48, "max_ms": 123.647, "p95_ms": 115.042}, "yolo11_640_fire_11class": {"avg_ms": 56.373, "count": 48, "max_ms": 82.557, "p95_ms": 75.728}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.962, "count": 48, "max_ms": 126.329, "p95_ms": 116.377}}, "truncated_count": 0}, "processing_total": {"avg_ms": 519.518, "count": 49, "max_ms": 891.41, "p95_ms": 749.482}, "read_interval": {"avg_ms": 527.066, "count": 48, "max_ms": 897.433, "p95_ms": 757.056}, "triton_wait": {"avg_ms": 91.338, "count": 48, "max_ms": 126.342, "p95_ms": 116.394}, "visualize": {"avg_ms": 19.523, "count": 48, "max_ms": 29.075, "p95_ms": 27.476}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 111}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4782.051, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e3c713dc40] error while decoding MB 44 29, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:56:51+09:00", "last_error_summary": "[h264 @ 0x55e3c713dc40] error while decoding MB 44 29, bytestream -5", "last_exit_at": "2026-07-06T11:54:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.195, "last_started_at": "2026-07-06T11:54:35+09:00", "pid": 3313, "process": {"exists": false, "pid": 3313}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 536.843, "sample_count": 685, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13375, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:56:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-06T11:56:51+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T11:56:51+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T11:56:51.879702+09:00", "mtime_epoch": 1783306611.8797023, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:56:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1347.543, "name": "celery", "pid": 237, "rss_mb": 1221.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 62, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.094, "pushed": 2.094, "read": 2.094}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.395, "count": 62, "max_ms": 33.226, "p95_ms": 27.725}, "gstreamer_push": {"avg_ms": 0.889, "count": 62, "max_ms": 1.575, "p95_ms": 0.974}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.337, "count": 62, "max_ms": 71.171, "p95_ms": 63.141}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 95.586, "count": 62, "max_ms": 126.414, "p95_ms": 123.642}, "yolo11_640_fire_11class": {"avg_ms": 56.678, "count": 62, "max_ms": 81.976, "p95_ms": 76.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 86.617, "count": 62, "max_ms": 121.954, "p95_ms": 115.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 477.627, "count": 62, "max_ms": 567.601, "p95_ms": 539.845}, "read_interval": {"avg_ms": 484.59, "count": 62, "max_ms": 575.716, "p95_ms": 548.864}, "triton_wait": {"avg_ms": 95.773, "count": 62, "max_ms": 126.427, "p95_ms": 123.656}, "visualize": {"avg_ms": 18.281, "count": 62, "max_ms": 27.134, "p95_ms": 20.659}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.61, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 17, "tracking_summary": 111}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4939.406, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:56:51+09:00", "last_error_summary": "[h264 @ 0x55e3c713dc40] error while decoding MB 44 29, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:56:51+09:00", "last_error_summary": "[h264 @ 0x55e3c713dc40] error while decoding MB 44 29, bytestream -5", "last_exit_at": "2026-07-06T11:56:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.072, "last_started_at": "2026-07-06T11:54:35+09:00", "pid": 3313, "process": {"exists": false, "pid": 3313}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 537.933, "sample_count": 682, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13376, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:56:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T11:56:52+09:00", "last_frame_read_age_sec": 0.927, "last_frame_read_at": "2026-07-06T11:56:51+09:00", "mjpeg": {"age_sec": 0.916, "exists": true, "mtime": "2026-07-06T11:56:51.879702+09:00", "mtime_epoch": 1783306611.8797023, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:56:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1347.543, "name": "celery", "pid": 237, "rss_mb": 1221.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 0.783, "recorders": [{"age_sec": 0.783, "frame_count": 51, "is_recording": true, "is_stopping": false, "pid": 3420, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.055, "name": "ffmpeg", "pid": 3420, "rss_mb": 187.055, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 49, "returncode": null, "running": true, "started_at": "2026-07-06T11:56:52+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_115651_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [3420], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.063, "pushed": 2.063, "read": 2.029}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.383, "count": 61, "max_ms": 33.226, "p95_ms": 27.725}, "gstreamer_push": {"avg_ms": 0.892, "count": 61, "max_ms": 1.575, "p95_ms": 0.974}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.448, "count": 61, "max_ms": 71.171, "p95_ms": 63.141}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 95.423, "count": 61, "max_ms": 126.414, "p95_ms": 123.642}, "yolo11_640_fire_11class": {"avg_ms": 56.499, "count": 61, "max_ms": 81.976, "p95_ms": 76.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 86.494, "count": 61, "max_ms": 121.954, "p95_ms": 115.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 485.278, "count": 61, "max_ms": 919.548, "p95_ms": 547.803}, "read_interval": {"avg_ms": 485.036, "count": 60, "max_ms": 575.716, "p95_ms": 548.864}, "triton_wait": {"avg_ms": 95.598, "count": 61, "max_ms": 126.427, "p95_ms": 123.656}, "visualize": {"avg_ms": 18.353, "count": 61, "max_ms": 27.134, "p95_ms": 20.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.568, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 17, "tracking_summary": 111}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4939.398, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:56:51+09:00", "last_error_summary": "[h264 @ 0x55e3c713dc40] error while decoding MB 44 29, bytestream -5", "last_exit_at": "2026-07-06T11:56:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.075, "last_started_at": "2026-07-06T11:54:35+09:00", "pid": 3313, "process": {"exists": false, "pid": 3313}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 537.933, "sample_count": 682, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13376, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:56:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T11:56:52+09:00", "last_frame_read_age_sec": 0.93, "last_frame_read_at": "2026-07-06T11:56:51+09:00", "mjpeg": {"age_sec": 0.919, "exists": true, "mtime": "2026-07-06T11:56:51.879702+09:00", "mtime_epoch": 1783306611.8797023, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:56:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1347.543, "name": "celery", "pid": 237, "rss_mb": 1221.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 0.786, "recorders": [{"age_sec": 0.786, "frame_count": 51, "is_recording": true, "is_stopping": false, "pid": 3420, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.055, "name": "ffmpeg", "pid": 3420, "rss_mb": 187.055, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 49, "returncode": null, "running": true, "started_at": "2026-07-06T11:56:52+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_115651_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [3420], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.063, "pushed": 2.063, "read": 2.029}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.383, "count": 61, "max_ms": 33.226, "p95_ms": 27.725}, "gstreamer_push": {"avg_ms": 0.892, "count": 61, "max_ms": 1.575, "p95_ms": 0.974}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.448, "count": 61, "max_ms": 71.171, "p95_ms": 63.141}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 95.423, "count": 61, "max_ms": 126.414, "p95_ms": 123.642}, "yolo11_640_fire_11class": {"avg_ms": 56.499, "count": 61, "max_ms": 81.976, "p95_ms": 76.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 86.494, "count": 61, "max_ms": 121.954, "p95_ms": 115.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 485.278, "count": 61, "max_ms": 919.548, "p95_ms": 547.803}, "read_interval": {"avg_ms": 485.036, "count": 60, "max_ms": 575.716, "p95_ms": 548.864}, "triton_wait": {"avg_ms": 95.598, "count": 61, "max_ms": 126.427, "p95_ms": 123.656}, "visualize": {"avg_ms": 18.353, "count": 61, "max_ms": 27.134, "p95_ms": 20.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.571, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 17, "tracking_summary": 111}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5046.074, "limit_mb": 8796093022207.996}, "detail": {"pid": 3480, "restart_count": 13, "start_count": 14}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:56:51+09:00", "last_error_summary": "[h264 @ 0x55e3c713dc40] error while decoding MB 44 29, bytestream -5", "last_exit_at": "2026-07-06T11:56:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:56:55+09:00", "pid": 3480, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.797, "name": "ffmpeg", "pid": 3480, "rss_mb": 138.797, "state": "S (sleeping)", "threads": 106}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 539.845, "sample_count": 664, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13376, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:56:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.702, "last_frame_pushed_at": "2026-07-06T11:56:52+09:00", "last_frame_read_age_sec": 5.623, "last_frame_read_at": "2026-07-06T11:56:51+09:00", "mjpeg": {"age_sec": 4.684, "exists": true, "mtime": "2026-07-06T11:56:52.807719+09:00", "mtime_epoch": 1783306612.8077188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:56:57+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1347.543, "name": "celery", "pid": 237, "rss_mb": 1221.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 5.48, "recorders": [{"age_sec": 5.48, "frame_count": 51, "is_recording": true, "is_stopping": false, "pid": 3420, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.055, "name": "ffmpeg", "pid": 3420, "rss_mb": 187.055, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 49, "returncode": null, "running": true, "started_at": "2026-07-06T11:56:52+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_115651_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [3420], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.718, "pushed": 1.718, "read": 1.685}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.683, "count": 51, "max_ms": 33.226, "p95_ms": 30.577}, "gstreamer_push": {"avg_ms": 0.896, "count": 51, "max_ms": 1.575, "p95_ms": 1.025}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.262, "count": 51, "max_ms": 71.171, "p95_ms": 67.468}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 96.772, "count": 51, "max_ms": 126.414, "p95_ms": 124.301}, "yolo11_640_fire_11class": {"avg_ms": 57.57, "count": 51, "max_ms": 81.976, "p95_ms": 77.288}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 87.156, "count": 51, "max_ms": 121.954, "p95_ms": 116.308}}, "truncated_count": 0}, "processing_total": {"avg_ms": 491.148, "count": 51, "max_ms": 919.548, "p95_ms": 548.297}, "read_interval": {"avg_ms": 489.545, "count": 50, "max_ms": 575.716, "p95_ms": 553.617}, "triton_wait": {"avg_ms": 96.845, "count": 51, "max_ms": 126.427, "p95_ms": 124.314}, "visualize": {"avg_ms": 18.481, "count": 51, "max_ms": 27.134, "p95_ms": 21.908}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.678, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5266.844, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5583c0f61780] error while decoding MB 39 46, bytestream -25 [in#0/rtsp @ 0x5583c0e46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583c0e46e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:57:18+09:00", "last_error_summary": "[h264 @ 0x5583c0f61780] error while decoding MB 39 46, bytestream -25 [in#0/rtsp @ 0x5583c0e46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583c0e46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:42:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 880.37, "last_started_at": "2026-07-06T11:42:37+09:00", "pid": 2600, "process": {"exists": false, "pid": 2600}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.305, "sample_count": 2298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19578, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:57:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T11:57:18+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T11:57:18+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T11:57:18.124167+09:00", "mtime_epoch": 1783306638.1241672, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228048}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:57:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1064.59, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 21, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.701, "pushed": 0.701, "read": 0.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.57, "count": 21, "max_ms": 6.356, "p95_ms": 5.696}, "gstreamer_push": {"avg_ms": 1.408, "count": 21, "max_ms": 6.418, "p95_ms": 1.396}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.912, "count": 21, "max_ms": 61.238, "p95_ms": 60.699}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.54, "count": 21, "max_ms": 111.759, "p95_ms": 104.955}, "yolo11_640_fire_11class": {"avg_ms": 60.174, "count": 21, "max_ms": 71.764, "p95_ms": 69.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.377, "count": 21, "max_ms": 102.389, "p95_ms": 90.148}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.404, "count": 21, "max_ms": 156.193, "p95_ms": 124.135}, "read_interval": {"avg_ms": 1429.25, "count": 21, "max_ms": 17747.625, "p95_ms": 10157.327}, "triton_wait": {"avg_ms": 71.614, "count": 21, "max_ms": 111.771, "p95_ms": 104.967}, "visualize": {"avg_ms": 4.184, "count": 21, "max_ms": 5.609, "p95_ms": 5.14}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5262.539, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:57:18+09:00", "last_error_summary": "[h264 @ 0x5583c0f61780] error while decoding MB 39 46, bytestream -25 [in#0/rtsp @ 0x5583c0e46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583c0e46e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:57:18+09:00", "last_error_summary": "[h264 @ 0x5583c0f61780] error while decoding MB 39 46, bytestream -25 [in#0/rtsp @ 0x5583c0e46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583c0e46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:57:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 880.439, "last_started_at": "2026-07-06T11:42:37+09:00", "pid": 2600, "process": {"exists": false, "pid": 2600}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.305, "sample_count": 2299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19579, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:57:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T11:57:18+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T11:57:18+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T11:57:18.124167+09:00", "mtime_epoch": 1783306638.1241672, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228048}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:57:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1064.59, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 20, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.702, "pushed": 0.702, "read": 0.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.746, "count": 21, "max_ms": 7.291, "p95_ms": 6.356}, "gstreamer_push": {"avg_ms": 1.393, "count": 21, "max_ms": 6.418, "p95_ms": 1.378}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.371, "count": 21, "max_ms": 60.699, "p95_ms": 60.488}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.357, "count": 21, "max_ms": 104.955, "p95_ms": 92.762}, "yolo11_640_fire_11class": {"avg_ms": 60.104, "count": 21, "max_ms": 71.764, "p95_ms": 69.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.996, "count": 21, "max_ms": 102.389, "p95_ms": 72.455}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.437, "count": 21, "max_ms": 156.193, "p95_ms": 124.834}, "read_interval": {"avg_ms": 1493.844, "count": 20, "max_ms": 17747.625, "p95_ms": 10157.327}, "triton_wait": {"avg_ms": 69.43, "count": 21, "max_ms": 104.967, "p95_ms": 92.773}, "visualize": {"avg_ms": 4.349, "count": 21, "max_ms": 7.159, "p95_ms": 5.609}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5262.508, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:57:18+09:00", "last_error_summary": "[h264 @ 0x5583c0f61780] error while decoding MB 39 46, bytestream -25 [in#0/rtsp @ 0x5583c0e46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583c0e46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:57:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 880.445, "last_started_at": "2026-07-06T11:42:37+09:00", "pid": 2600, "process": {"exists": false, "pid": 2600}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.305, "sample_count": 2299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19579, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:57:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T11:57:18+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T11:57:18+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-06T11:57:18.124167+09:00", "mtime_epoch": 1783306638.1241672, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228048}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:57:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1064.59, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 20, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.701, "pushed": 0.701, "read": 0.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.746, "count": 21, "max_ms": 7.291, "p95_ms": 6.356}, "gstreamer_push": {"avg_ms": 1.393, "count": 21, "max_ms": 6.418, "p95_ms": 1.378}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.371, "count": 21, "max_ms": 60.699, "p95_ms": 60.488}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.357, "count": 21, "max_ms": 104.955, "p95_ms": 92.762}, "yolo11_640_fire_11class": {"avg_ms": 60.104, "count": 21, "max_ms": 71.764, "p95_ms": 69.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.996, "count": 21, "max_ms": 102.389, "p95_ms": 72.455}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.437, "count": 21, "max_ms": 156.193, "p95_ms": 124.834}, "read_interval": {"avg_ms": 1493.844, "count": 20, "max_ms": 17747.625, "p95_ms": 10157.327}, "triton_wait": {"avg_ms": 69.43, "count": 21, "max_ms": 104.967, "p95_ms": 92.773}, "visualize": {"avg_ms": 4.349, "count": 21, "max_ms": 7.159, "p95_ms": 5.609}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5281.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 3589, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:57:18+09:00", "last_error_summary": "[h264 @ 0x5583c0f61780] error while decoding MB 39 46, bytestream -25 [in#0/rtsp @ 0x5583c0e46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5583c0e46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:57:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:57:20+09:00", "pid": 3589, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.559, "name": "ffmpeg", "pid": 3589, "rss_mb": 58.559, "state": "S (sleeping)", "threads": 1}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.509, "sample_count": 2269, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19579, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:57:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 4.084, "last_frame_pushed_at": "2026-07-06T11:57:18+09:00", "last_frame_read_age_sec": 4.21, "last_frame_read_at": "2026-07-06T11:57:18+09:00", "mjpeg": {"age_sec": 4.07, "exists": true, "mtime": "2026-07-06T11:57:18.256170+09:00", "mtime_epoch": 1783306638.2561696, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227883}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T11:57:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1064.59, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.827, "pushed": 0.827, "read": 0.827}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.369, "count": 13, "max_ms": 7.291, "p95_ms": 7.291}, "gstreamer_push": {"avg_ms": 1.081, "count": 13, "max_ms": 1.255, "p95_ms": 1.255}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.961, "count": 13, "max_ms": 60.488, "p95_ms": 60.488}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.267, "count": 13, "max_ms": 104.955, "p95_ms": 104.955}, "yolo11_640_fire_11class": {"avg_ms": 61.25, "count": 13, "max_ms": 69.569, "p95_ms": 69.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.125, "count": 13, "max_ms": 102.389, "p95_ms": 102.389}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.949, "count": 13, "max_ms": 156.193, "p95_ms": 156.193}, "read_interval": {"avg_ms": 2250.305, "count": 13, "max_ms": 17747.625, "p95_ms": 17747.625}, "triton_wait": {"avg_ms": 71.378, "count": 13, "max_ms": 104.967, "p95_ms": 104.967}, "visualize": {"avg_ms": 4.84, "count": 13, "max_ms": 7.159, "p95_ms": 7.159}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.717, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 24, "tracking_summary": 113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4803.094, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fa17c9ea40] error while decoding MB 12 41, bytestream -22", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:59:12+09:00", "last_error_summary": "[h264 @ 0x55fa17c9ea40] error while decoding MB 12 41, bytestream -22", "last_exit_at": "2026-07-06T11:56:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.711, "last_started_at": "2026-07-06T11:56:55+09:00", "pid": 3480, "process": {"exists": false, "pid": 3480}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 670.009, "sample_count": 567, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13617, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:59:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-06T11:59:12+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T11:59:12+09:00", "mjpeg": {"age_sec": 0.031, "exists": true, "mtime": "2026-07-06T11:59:12.170188+09:00", "mtime_epoch": 1783306752.170188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:59:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1224.266, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 60, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.014, "pushed": 2.014, "read": 2.014}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.022, "count": 59, "max_ms": 35.521, "p95_ms": 32.259}, "gstreamer_push": {"avg_ms": 0.866, "count": 60, "max_ms": 1.024, "p95_ms": 0.951}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.189, "count": 59, "max_ms": 97.985, "p95_ms": 80.777}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 101.263, "count": 59, "max_ms": 135.258, "p95_ms": 127.791}, "yolo11_640_fire_11class": {"avg_ms": 61.931, "count": 59, "max_ms": 98.549, "p95_ms": 81.093}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.599, "count": 59, "max_ms": 128.514, "p95_ms": 123.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 496.619, "count": 60, "max_ms": 565.479, "p95_ms": 552.637}, "read_interval": {"avg_ms": 503.33, "count": 60, "max_ms": 571.446, "p95_ms": 558.369}, "triton_wait": {"avg_ms": 101.793, "count": 59, "max_ms": 135.27, "p95_ms": 127.804}, "visualize": {"avg_ms": 21.711, "count": 59, "max_ms": 31.062, "p95_ms": 27.097}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.786, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 24, "tracking_summary": 113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4806.156, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T11:59:12+09:00", "last_error_summary": "[h264 @ 0x55fa17c9ea40] error while decoding MB 12 41, bytestream -22", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:59:12+09:00", "last_error_summary": "[h264 @ 0x55fa17c9ea40] error while decoding MB 12 41, bytestream -22", "last_exit_at": "2026-07-06T11:59:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.094, "last_started_at": "2026-07-06T11:56:55+09:00", "pid": 3480, "process": {"exists": false, "pid": 3480}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 670.009, "sample_count": 567, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13618, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:59:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T11:59:12+09:00", "last_frame_read_age_sec": 0.435, "last_frame_read_at": "2026-07-06T11:59:12+09:00", "mjpeg": {"age_sec": 0.414, "exists": true, "mtime": "2026-07-06T11:59:12.170188+09:00", "mtime_epoch": 1783306752.170188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:59:12+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1224.266, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.02, "pushed": 2.02, "read": 1.986}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.935, "count": 59, "max_ms": 35.521, "p95_ms": 32.259}, "gstreamer_push": {"avg_ms": 0.866, "count": 60, "max_ms": 1.024, "p95_ms": 0.951}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.146, "count": 59, "max_ms": 97.985, "p95_ms": 80.777}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 101.302, "count": 59, "max_ms": 135.258, "p95_ms": 127.791}, "yolo11_640_fire_11class": {"avg_ms": 62.079, "count": 59, "max_ms": 98.549, "p95_ms": 81.093}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.65, "count": 59, "max_ms": 128.514, "p95_ms": 123.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 496.02, "count": 60, "max_ms": 565.479, "p95_ms": 552.637}, "read_interval": {"avg_ms": 503.855, "count": 59, "max_ms": 571.446, "p95_ms": 564.927}, "triton_wait": {"avg_ms": 101.831, "count": 59, "max_ms": 135.27, "p95_ms": 127.804}, "visualize": {"avg_ms": 21.635, "count": 59, "max_ms": 31.062, "p95_ms": 27.097}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.709, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 24, "tracking_summary": 113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4806.148, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:59:12+09:00", "last_error_summary": "[h264 @ 0x55fa17c9ea40] error while decoding MB 12 41, bytestream -22", "last_exit_at": "2026-07-06T11:59:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.097, "last_started_at": "2026-07-06T11:56:55+09:00", "pid": 3480, "process": {"exists": false, "pid": 3480}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 670.009, "sample_count": 567, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13618, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:59:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T11:59:12+09:00", "last_frame_read_age_sec": 0.437, "last_frame_read_at": "2026-07-06T11:59:12+09:00", "mjpeg": {"age_sec": 0.417, "exists": true, "mtime": "2026-07-06T11:59:12.170188+09:00", "mtime_epoch": 1783306752.170188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:59:12+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1224.266, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.019, "pushed": 2.019, "read": 1.986}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.935, "count": 59, "max_ms": 35.521, "p95_ms": 32.259}, "gstreamer_push": {"avg_ms": 0.866, "count": 60, "max_ms": 1.024, "p95_ms": 0.951}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.146, "count": 59, "max_ms": 97.985, "p95_ms": 80.777}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 101.302, "count": 59, "max_ms": 135.258, "p95_ms": 127.791}, "yolo11_640_fire_11class": {"avg_ms": 62.079, "count": 59, "max_ms": 98.549, "p95_ms": 81.093}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.65, "count": 59, "max_ms": 128.514, "p95_ms": 123.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 496.02, "count": 60, "max_ms": 565.479, "p95_ms": 552.637}, "read_interval": {"avg_ms": 503.855, "count": 59, "max_ms": 571.446, "p95_ms": 564.927}, "triton_wait": {"avg_ms": 101.831, "count": 59, "max_ms": 135.27, "p95_ms": 127.804}, "visualize": {"avg_ms": 21.635, "count": 59, "max_ms": 31.062, "p95_ms": 27.097}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.711, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 24, "tracking_summary": 113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4913.609, "limit_mb": 8796093022207.996}, "detail": {"pid": 3741, "restart_count": 14, "start_count": 15}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T11:59:12+09:00", "last_error_summary": "[h264 @ 0x55fa17c9ea40] error while decoding MB 12 41, bytestream -22", "last_exit_at": "2026-07-06T11:59:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T11:59:14+09:00", "pid": 3741, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.012, "name": "ffmpeg", "pid": 3741, "rss_mb": 139.012, "state": "S (sleeping)", "threads": 106}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 670.009, "sample_count": 559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13618, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T11:59:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.059, "last_frame_pushed_at": "2026-07-06T11:59:12+09:00", "last_frame_read_age_sec": 4.489, "last_frame_read_at": "2026-07-06T11:59:12+09:00", "mjpeg": {"age_sec": 4.041, "exists": true, "mtime": "2026-07-06T11:59:12.598196+09:00", "mtime_epoch": 1783306752.5981956, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198135}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T11:59:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1224.266, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.757, "pushed": 1.757, "read": 1.724}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.856, "count": 52, "max_ms": 35.521, "p95_ms": 32.259}, "gstreamer_push": {"avg_ms": 0.867, "count": 52, "max_ms": 1.024, "p95_ms": 0.955}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.917, "count": 52, "max_ms": 97.985, "p95_ms": 80.777}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 100.998, "count": 52, "max_ms": 135.258, "p95_ms": 127.791}, "yolo11_640_fire_11class": {"avg_ms": 62.657, "count": 52, "max_ms": 98.549, "p95_ms": 81.093}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.894, "count": 52, "max_ms": 128.514, "p95_ms": 123.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 494.318, "count": 52, "max_ms": 565.479, "p95_ms": 556.903}, "read_interval": {"avg_ms": 502.34, "count": 51, "max_ms": 571.446, "p95_ms": 564.927}, "triton_wait": {"avg_ms": 101.558, "count": 52, "max_ms": 135.27, "p95_ms": 127.804}, "visualize": {"avg_ms": 21.159, "count": 52, "max_ms": 28.54, "p95_ms": 26.233}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.59, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 115}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4748.953, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5594c42cd140] error while decoding MB 59 35, bytestream -24", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:01:25+09:00", "last_error_summary": "[h264 @ 0x5594c42cd140] error while decoding MB 59 35, bytestream -24", "last_exit_at": "2026-07-06T11:59:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.709, "last_started_at": "2026-07-06T11:59:14+09:00", "pid": 3741, "process": {"exists": false, "pid": 3741}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 662.947, "sample_count": 573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13893, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:01:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T12:01:25+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T12:01:25+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-06T12:01:25.312549+09:00", "mtime_epoch": 1783306885.312549, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209481}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:01:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.238, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 66, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.231, "pushed": 2.231, "read": 2.231}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.925, "count": 66, "max_ms": 28.348, "p95_ms": 26.744}, "gstreamer_push": {"avg_ms": 0.879, "count": 66, "max_ms": 1.317, "p95_ms": 0.972}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.625, "count": 66, "max_ms": 85.365, "p95_ms": 72.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 98.282, "count": 66, "max_ms": 128.647, "p95_ms": 122.117}, "yolo11_640_fire_11class": {"avg_ms": 59.87, "count": 66, "max_ms": 85.875, "p95_ms": 78.264}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.321, "count": 66, "max_ms": 128.66, "p95_ms": 122.531}}, "truncated_count": 0}, "processing_total": {"avg_ms": 447.202, "count": 66, "max_ms": 524.12, "p95_ms": 490.503}, "read_interval": {"avg_ms": 453.832, "count": 66, "max_ms": 529.579, "p95_ms": 500.294}, "triton_wait": {"avg_ms": 98.963, "count": 66, "max_ms": 128.671, "p95_ms": 122.542}, "visualize": {"avg_ms": 17.931, "count": 66, "max_ms": 25.991, "p95_ms": 19.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.577, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 115}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4746.742, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:01:25+09:00", "last_error_summary": "[h264 @ 0x5594c42cd140] error while decoding MB 59 35, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:01:25+09:00", "last_error_summary": "[h264 @ 0x5594c42cd140] error while decoding MB 59 35, bytestream -24", "last_exit_at": "2026-07-06T12:01:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.11, "last_started_at": "2026-07-06T11:59:14+09:00", "pid": 3741, "process": {"exists": false, "pid": 3741}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 662.947, "sample_count": 573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13894, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:01:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-06T12:01:25+09:00", "last_frame_read_age_sec": 0.454, "last_frame_read_at": "2026-07-06T12:01:25+09:00", "mjpeg": {"age_sec": 0.436, "exists": true, "mtime": "2026-07-06T12:01:25.312549+09:00", "mtime_epoch": 1783306885.312549, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209481}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:01:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.238, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.235, "pushed": 2.235, "read": 2.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.945, "count": 66, "max_ms": 28.348, "p95_ms": 26.744}, "gstreamer_push": {"avg_ms": 0.879, "count": 67, "max_ms": 1.317, "p95_ms": 0.972}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.8, "count": 66, "max_ms": 85.365, "p95_ms": 72.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 98.538, "count": 66, "max_ms": 128.647, "p95_ms": 122.117}, "yolo11_640_fire_11class": {"avg_ms": 60.054, "count": 66, "max_ms": 85.875, "p95_ms": 78.264}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.582, "count": 66, "max_ms": 128.66, "p95_ms": 122.531}}, "truncated_count": 0}, "processing_total": {"avg_ms": 447.241, "count": 67, "max_ms": 524.12, "p95_ms": 490.503}, "read_interval": {"avg_ms": 453.832, "count": 66, "max_ms": 529.579, "p95_ms": 500.294}, "triton_wait": {"avg_ms": 99.219, "count": 66, "max_ms": 128.671, "p95_ms": 122.542}, "visualize": {"avg_ms": 17.928, "count": 66, "max_ms": 25.991, "p95_ms": 19.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 115}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4746.734, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:01:25+09:00", "last_error_summary": "[h264 @ 0x5594c42cd140] error while decoding MB 59 35, bytestream -24", "last_exit_at": "2026-07-06T12:01:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.113, "last_started_at": "2026-07-06T11:59:14+09:00", "pid": 3741, "process": {"exists": false, "pid": 3741}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 662.947, "sample_count": 573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13894, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:01:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T12:01:25+09:00", "last_frame_read_age_sec": 0.457, "last_frame_read_at": "2026-07-06T12:01:25+09:00", "mjpeg": {"age_sec": 0.439, "exists": true, "mtime": "2026-07-06T12:01:25.312549+09:00", "mtime_epoch": 1783306885.312549, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209481}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:01:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.238, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.235, "pushed": 2.235, "read": 2.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.945, "count": 66, "max_ms": 28.348, "p95_ms": 26.744}, "gstreamer_push": {"avg_ms": 0.879, "count": 67, "max_ms": 1.317, "p95_ms": 0.972}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.8, "count": 66, "max_ms": 85.365, "p95_ms": 72.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 98.538, "count": 66, "max_ms": 128.647, "p95_ms": 122.117}, "yolo11_640_fire_11class": {"avg_ms": 60.054, "count": 66, "max_ms": 85.875, "p95_ms": 78.264}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.582, "count": 66, "max_ms": 128.66, "p95_ms": 122.531}}, "truncated_count": 0}, "processing_total": {"avg_ms": 447.241, "count": 67, "max_ms": 524.12, "p95_ms": 490.503}, "read_interval": {"avg_ms": 453.832, "count": 66, "max_ms": 529.579, "p95_ms": 500.294}, "triton_wait": {"avg_ms": 99.219, "count": 66, "max_ms": 128.671, "p95_ms": 122.542}, "visualize": {"avg_ms": 17.928, "count": 66, "max_ms": 25.991, "p95_ms": 19.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 115}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4864.012, "limit_mb": 8796093022207.996}, "detail": {"pid": 3908, "restart_count": 15, "start_count": 16}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:01:25+09:00", "last_error_summary": "[h264 @ 0x5594c42cd140] error while decoding MB 59 35, bytestream -24", "last_exit_at": "2026-07-06T12:01:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:01:28+09:00", "pid": 3908, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.902, "name": "ffmpeg", "pid": 3908, "rss_mb": 141.902, "state": "S (sleeping)", "threads": 106}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 663.277, "sample_count": 563, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13894, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:01:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.694, "last_frame_pushed_at": "2026-07-06T12:01:25+09:00", "last_frame_read_age_sec": 5.145, "last_frame_read_at": "2026-07-06T12:01:25+09:00", "mjpeg": {"age_sec": 4.68, "exists": true, "mtime": "2026-07-06T12:01:25.760557+09:00", "mtime_epoch": 1783306885.760557, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209933}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:01:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.238, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.872, "pushed": 1.872, "read": 1.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.063, "count": 55, "max_ms": 28.348, "p95_ms": 26.968}, "gstreamer_push": {"avg_ms": 0.878, "count": 56, "max_ms": 1.317, "p95_ms": 0.972}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.326, "count": 55, "max_ms": 85.365, "p95_ms": 72.862}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 100.428, "count": 55, "max_ms": 128.647, "p95_ms": 125.303}, "yolo11_640_fire_11class": {"avg_ms": 61.318, "count": 55, "max_ms": 85.875, "p95_ms": 80.226}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 96.759, "count": 55, "max_ms": 128.66, "p95_ms": 126.579}}, "truncated_count": 0}, "processing_total": {"avg_ms": 450.922, "count": 56, "max_ms": 524.12, "p95_ms": 494.383}, "read_interval": {"avg_ms": 457.541, "count": 55, "max_ms": 529.579, "p95_ms": 500.341}, "triton_wait": {"avg_ms": 101.2, "count": 55, "max_ms": 128.671, "p95_ms": 126.589}, "visualize": {"avg_ms": 18.016, "count": 55, "max_ms": 25.991, "p95_ms": 19.996}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4687.988, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55858b708fc0] error while decoding MB 118 29, bytestream -31 [in#0/rtsp @ 0x55858b568e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55858b568e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:02:13+09:00", "last_error_summary": "[h264 @ 0x55858b708fc0] error while decoding MB 118 29, bytestream -31 [in#0/rtsp @ 0x55858b568e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55858b568e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T11:57:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 292.714, "last_started_at": "2026-07-06T11:57:20+09:00", "pid": 3589, "process": {"exists": false, "pid": 3589}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.848, "sample_count": 2490, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22057, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:02:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T12:02:12+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T12:02:12+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T12:02:12.985395+09:00", "mtime_epoch": 1783306932.985395, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 209936}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:02:13+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1067.004, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 59, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.971, "pushed": 1.971, "read": 1.971}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.78, "count": 59, "max_ms": 8.17, "p95_ms": 7.351}, "gstreamer_push": {"avg_ms": 1.492, "count": 59, "max_ms": 6.076, "p95_ms": 3.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.273, "count": 59, "max_ms": 76.25, "p95_ms": 65.593}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.636, "count": 59, "max_ms": 123.846, "p95_ms": 115.532}, "yolo11_640_fire_11class": {"avg_ms": 60.177, "count": 59, "max_ms": 82.028, "p95_ms": 72.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.129, "count": 59, "max_ms": 117.677, "p95_ms": 109.979}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.338, "count": 59, "max_ms": 176.564, "p95_ms": 159.876}, "read_interval": {"avg_ms": 508.344, "count": 59, "max_ms": 14890.977, "p95_ms": 186.206}, "triton_wait": {"avg_ms": 71.799, "count": 59, "max_ms": 123.859, "p95_ms": 117.097}, "visualize": {"avg_ms": 4.101, "count": 59, "max_ms": 5.688, "p95_ms": 5.573}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4687.641, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:02:13+09:00", "last_error_summary": "[h264 @ 0x55858b708fc0] error while decoding MB 118 29, bytestream -31 [in#0/rtsp @ 0x55858b568e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55858b568e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:02:13+09:00", "last_error_summary": "[h264 @ 0x55858b708fc0] error while decoding MB 118 29, bytestream -31 [in#0/rtsp @ 0x55858b568e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55858b568e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T12:02:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 292.747, "last_started_at": "2026-07-06T11:57:20+09:00", "pid": 3589, "process": {"exists": false, "pid": 3589}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.848, "sample_count": 2491, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22058, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:02:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T12:02:13+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-06T12:02:12+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-06T12:02:12.985395+09:00", "mtime_epoch": 1783306932.985395, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 209936}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:02:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1067.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.002, "pushed": 2.002, "read": 1.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.734, "count": 59, "max_ms": 8.17, "p95_ms": 7.351}, "gstreamer_push": {"avg_ms": 1.488, "count": 60, "max_ms": 6.076, "p95_ms": 3.822}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.319, "count": 59, "max_ms": 76.25, "p95_ms": 65.593}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.808, "count": 59, "max_ms": 123.846, "p95_ms": 115.532}, "yolo11_640_fire_11class": {"avg_ms": 60.427, "count": 59, "max_ms": 82.028, "p95_ms": 72.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.272, "count": 59, "max_ms": 117.677, "p95_ms": 109.979}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.135, "count": 60, "max_ms": 176.564, "p95_ms": 142.922}, "read_interval": {"avg_ms": 508.344, "count": 59, "max_ms": 14890.977, "p95_ms": 186.206}, "triton_wait": {"avg_ms": 71.941, "count": 59, "max_ms": 123.859, "p95_ms": 117.097}, "visualize": {"avg_ms": 4.095, "count": 59, "max_ms": 5.688, "p95_ms": 5.573}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4687.805, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:02:13+09:00", "last_error_summary": "[h264 @ 0x55858b708fc0] error while decoding MB 118 29, bytestream -31 [in#0/rtsp @ 0x55858b568e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55858b568e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T12:02:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 292.754, "last_started_at": "2026-07-06T11:57:20+09:00", "pid": 3589, "process": {"exists": false, "pid": 3589}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.848, "sample_count": 2491, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22058, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:02:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-06T12:02:13+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-06T12:02:12+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-06T12:02:12.985395+09:00", "mtime_epoch": 1783306932.985395, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 209936}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:02:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1067.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.001, "pushed": 2.001, "read": 1.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.734, "count": 59, "max_ms": 8.17, "p95_ms": 7.351}, "gstreamer_push": {"avg_ms": 1.488, "count": 60, "max_ms": 6.076, "p95_ms": 3.822}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.319, "count": 59, "max_ms": 76.25, "p95_ms": 65.593}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.808, "count": 59, "max_ms": 123.846, "p95_ms": 115.532}, "yolo11_640_fire_11class": {"avg_ms": 60.427, "count": 59, "max_ms": 82.028, "p95_ms": 72.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.272, "count": 59, "max_ms": 117.677, "p95_ms": 109.979}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.135, "count": 60, "max_ms": 176.564, "p95_ms": 142.922}, "read_interval": {"avg_ms": 508.344, "count": 59, "max_ms": 14890.977, "p95_ms": 186.206}, "triton_wait": {"avg_ms": 71.941, "count": 59, "max_ms": 123.859, "p95_ms": 117.097}, "visualize": {"avg_ms": 4.095, "count": 59, "max_ms": 5.688, "p95_ms": 5.573}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4762.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 4015, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:02:13+09:00", "last_error_summary": "[h264 @ 0x55858b708fc0] error while decoding MB 118 29, bytestream -31 [in#0/rtsp @ 0x55858b568e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55858b568e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T12:02:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:02:15+09:00", "pid": 4015, "process": {"exists": true, "fd_count": 5, "hwm_mb": 107.027, "name": "ffmpeg", "pid": 4015, "rss_mb": 107.027, "state": "S (sleeping)", "threads": 19}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.6, "sample_count": 2485, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22058, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:02:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 4.513, "last_frame_pushed_at": "2026-07-06T12:02:13+09:00", "last_frame_read_age_sec": 4.599, "last_frame_read_at": "2026-07-06T12:02:12+09:00", "mjpeg": {"age_sec": 4.497, "exists": true, "mtime": "2026-07-06T12:02:13.077396+09:00", "mtime_epoch": 1783306933.0773964, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 209825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:02:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1067.004, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 18, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.635, "pushed": 0.635, "read": 0.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.553, "count": 19, "max_ms": 8.17, "p95_ms": 8.17}, "gstreamer_push": {"avg_ms": 1.578, "count": 19, "max_ms": 4.251, "p95_ms": 4.251}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.031, "count": 19, "max_ms": 55.251, "p95_ms": 55.251}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.477, "count": 19, "max_ms": 91.281, "p95_ms": 91.281}, "yolo11_640_fire_11class": {"avg_ms": 59.8, "count": 19, "max_ms": 70.038, "p95_ms": 70.038}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.147, "count": 19, "max_ms": 74.302, "p95_ms": 74.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.672, "count": 19, "max_ms": 131.901, "p95_ms": 131.901}, "read_interval": {"avg_ms": 1411.462, "count": 18, "max_ms": 14890.977, "p95_ms": 14890.977}, "triton_wait": {"avg_ms": 67.49, "count": 19, "max_ms": 91.293, "p95_ms": 91.293}, "visualize": {"avg_ms": 3.967, "count": 19, "max_ms": 5.41, "p95_ms": 5.41}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 27, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4751.434, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558a8e8c0540] error while decoding MB 34 14, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:03:43+09:00", "last_error_summary": "[h264 @ 0x558a8e8c0540] error while decoding MB 34 14, bytestream -7", "last_exit_at": "2026-07-06T12:01:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.446, "last_started_at": "2026-07-06T12:01:28+09:00", "pid": 3908, "process": {"exists": false, "pid": 3908}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 549.22, "sample_count": 598, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14158, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.053, "last_frame_pushed_at": "2026-07-06T12:03:43+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-06T12:03:43+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-06T12:03:43.851007+09:00", "mtime_epoch": 1783307023.8510075, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 218155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:03:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.238, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 62, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.081, "pushed": 2.081, "read": 2.081}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.713, "count": 61, "max_ms": 32.539, "p95_ms": 30.181}, "gstreamer_push": {"avg_ms": 0.901, "count": 62, "max_ms": 1.262, "p95_ms": 1.113}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.497, "count": 61, "max_ms": 85.685, "p95_ms": 79.892}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 102.867, "count": 61, "max_ms": 132.662, "p95_ms": 127.468}, "yolo11_640_fire_11class": {"avg_ms": 62.617, "count": 61, "max_ms": 97.331, "p95_ms": 88.668}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 100.424, "count": 61, "max_ms": 134.837, "p95_ms": 129.995}}, "truncated_count": 0}, "processing_total": {"avg_ms": 479.36, "count": 62, "max_ms": 555.808, "p95_ms": 517.36}, "read_interval": {"avg_ms": 486.378, "count": 62, "max_ms": 565.404, "p95_ms": 525.96}, "triton_wait": {"avg_ms": 104.475, "count": 61, "max_ms": 134.851, "p95_ms": 130.504}, "visualize": {"avg_ms": 18.724, "count": 61, "max_ms": 24.831, "p95_ms": 20.596}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.79, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 27, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4753.461, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:03:43+09:00", "last_error_summary": "[h264 @ 0x558a8e8c0540] error while decoding MB 34 14, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:03:43+09:00", "last_error_summary": "[h264 @ 0x558a8e8c0540] error while decoding MB 34 14, bytestream -7", "last_exit_at": "2026-07-06T12:03:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.864, "last_started_at": "2026-07-06T12:01:28+09:00", "pid": 3908, "process": {"exists": false, "pid": 3908}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 549.22, "sample_count": 598, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14159, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T12:03:44+09:00", "last_frame_read_age_sec": 0.467, "last_frame_read_at": "2026-07-06T12:03:43+09:00", "mjpeg": {"age_sec": 0.453, "exists": true, "mtime": "2026-07-06T12:03:43.851007+09:00", "mtime_epoch": 1783307023.8510075, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 218155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:03:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.238, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.085, "pushed": 2.085, "read": 2.051}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.817, "count": 61, "max_ms": 32.539, "p95_ms": 30.181}, "gstreamer_push": {"avg_ms": 0.901, "count": 62, "max_ms": 1.262, "p95_ms": 1.113}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.296, "count": 61, "max_ms": 85.685, "p95_ms": 79.892}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 102.731, "count": 61, "max_ms": 132.662, "p95_ms": 127.468}, "yolo11_640_fire_11class": {"avg_ms": 62.58, "count": 61, "max_ms": 97.331, "p95_ms": 88.668}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 100.373, "count": 61, "max_ms": 134.837, "p95_ms": 129.995}}, "truncated_count": 0}, "processing_total": {"avg_ms": 480.108, "count": 62, "max_ms": 555.808, "p95_ms": 517.36}, "read_interval": {"avg_ms": 487.45, "count": 61, "max_ms": 565.404, "p95_ms": 525.96}, "triton_wait": {"avg_ms": 104.384, "count": 61, "max_ms": 134.851, "p95_ms": 130.504}, "visualize": {"avg_ms": 18.749, "count": 61, "max_ms": 24.831, "p95_ms": 20.596}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.742, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 27, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4753.453, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:03:43+09:00", "last_error_summary": "[h264 @ 0x558a8e8c0540] error while decoding MB 34 14, bytestream -7", "last_exit_at": "2026-07-06T12:03:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.867, "last_started_at": "2026-07-06T12:01:28+09:00", "pid": 3908, "process": {"exists": false, "pid": 3908}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 549.22, "sample_count": 598, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14159, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T12:03:44+09:00", "last_frame_read_age_sec": 0.47, "last_frame_read_at": "2026-07-06T12:03:43+09:00", "mjpeg": {"age_sec": 0.456, "exists": true, "mtime": "2026-07-06T12:03:43.851007+09:00", "mtime_epoch": 1783307023.8510075, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 218155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:03:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.238, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.084, "pushed": 2.084, "read": 2.051}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.817, "count": 61, "max_ms": 32.539, "p95_ms": 30.181}, "gstreamer_push": {"avg_ms": 0.901, "count": 62, "max_ms": 1.262, "p95_ms": 1.113}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.296, "count": 61, "max_ms": 85.685, "p95_ms": 79.892}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 102.731, "count": 61, "max_ms": 132.662, "p95_ms": 127.468}, "yolo11_640_fire_11class": {"avg_ms": 62.58, "count": 61, "max_ms": 97.331, "p95_ms": 88.668}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 100.373, "count": 61, "max_ms": 134.837, "p95_ms": 129.995}}, "truncated_count": 0}, "processing_total": {"avg_ms": 480.108, "count": 62, "max_ms": 555.808, "p95_ms": 517.36}, "read_interval": {"avg_ms": 487.45, "count": 61, "max_ms": 565.404, "p95_ms": 525.96}, "triton_wait": {"avg_ms": 104.384, "count": 61, "max_ms": 134.851, "p95_ms": 130.504}, "visualize": {"avg_ms": 18.749, "count": 61, "max_ms": 24.831, "p95_ms": 20.596}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.744, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 27, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4862.477, "limit_mb": 8796093022207.996}, "detail": {"pid": 4227, "restart_count": 16, "start_count": 17}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:03:43+09:00", "last_error_summary": "[h264 @ 0x558a8e8c0540] error while decoding MB 34 14, bytestream -7", "last_exit_at": "2026-07-06T12:03:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:03:46+09:00", "pid": 4227, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.492, "name": "ffmpeg", "pid": 4227, "rss_mb": 139.492, "state": "S (sleeping)", "threads": 106}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 549.22, "sample_count": 590, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14159, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.201, "last_frame_pushed_at": "2026-07-06T12:03:44+09:00", "last_frame_read_age_sec": 4.664, "last_frame_read_at": "2026-07-06T12:03:43+09:00", "mjpeg": {"age_sec": 4.182, "exists": true, "mtime": "2026-07-06T12:03:44.319016+09:00", "mtime_epoch": 1783307024.3190157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 218007}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:03:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.238, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.779, "pushed": 1.779, "read": 1.746}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.223, "count": 52, "max_ms": 32.539, "p95_ms": 30.267}, "gstreamer_push": {"avg_ms": 0.899, "count": 53, "max_ms": 1.262, "p95_ms": 1.137}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.309, "count": 52, "max_ms": 85.685, "p95_ms": 80.438}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 103.535, "count": 52, "max_ms": 132.662, "p95_ms": 130.192}, "yolo11_640_fire_11class": {"avg_ms": 63.121, "count": 52, "max_ms": 97.331, "p95_ms": 89.328}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 101.075, "count": 52, "max_ms": 134.837, "p95_ms": 129.995}}, "truncated_count": 0}, "processing_total": {"avg_ms": 483.996, "count": 53, "max_ms": 555.808, "p95_ms": 526.899}, "read_interval": {"avg_ms": 491.503, "count": 52, "max_ms": 565.404, "p95_ms": 534.461}, "triton_wait": {"avg_ms": 105.261, "count": 52, "max_ms": 134.851, "p95_ms": 130.904}, "visualize": {"avg_ms": 18.89, "count": 52, "max_ms": 24.831, "p95_ms": 21.174}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.787, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4694.059, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55db63165200] missing picture in access unit with size 18 [h264 @ 0x55db631f4700] no frame! [in#0/rtsp @ 0x55db63160e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55db63160e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55db63165840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:04:29+09:00", "last_error_summary": "[NULL @ 0x55db63165200] missing picture in access unit with size 18 [h264 @ 0x55db631f4700] no frame! [in#0/rtsp @ 0x55db63160e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55db63160e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55db63165840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T12:02:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.47, "last_started_at": "2026-07-06T12:02:15+09:00", "pid": 4015, "process": {"exists": false, "pid": 4015}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.689, "sample_count": 2151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22955, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:04:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-06T12:04:28+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-06T12:04:28+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T12:04:28.995809+09:00", "mtime_epoch": 1783307068.9958088, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218096}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:04:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1066.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.474, "pushed": 0.474, "read": 0.511}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.987, "count": 13, "max_ms": 6.833, "p95_ms": 6.833}, "gstreamer_push": {"avg_ms": 1.13, "count": 13, "max_ms": 1.346, "p95_ms": 1.346}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.331, "count": 13, "max_ms": 58.424, "p95_ms": 58.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.499, "count": 13, "max_ms": 106.093, "p95_ms": 106.093}, "yolo11_640_fire_11class": {"avg_ms": 58.647, "count": 13, "max_ms": 65.705, "p95_ms": 65.705}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.149, "count": 13, "max_ms": 105.593, "p95_ms": 105.593}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.468, "count": 13, "max_ms": 161.332, "p95_ms": 161.332}, "read_interval": {"avg_ms": 2386.148, "count": 14, "max_ms": 26006.235, "p95_ms": 26006.235}, "triton_wait": {"avg_ms": 68.889, "count": 13, "max_ms": 106.103, "p95_ms": 106.103}, "visualize": {"avg_ms": 4.314, "count": 13, "max_ms": 5.421, "p95_ms": 5.421}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.421, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4693.281, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:04:29+09:00", "last_error_summary": "[NULL @ 0x55db63165200] missing picture in access unit with size 18 [h264 @ 0x55db631f4700] no frame! [in#0/rtsp @ 0x55db63160e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55db63160e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55db63165840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:04:29+09:00", "last_error_summary": "[NULL @ 0x55db63165200] missing picture in access unit with size 18 [h264 @ 0x55db631f4700] no frame! [in#0/rtsp @ 0x55db63160e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55db63160e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55db63165840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T12:04:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.536, "last_started_at": "2026-07-06T12:02:15+09:00", "pid": 4015, "process": {"exists": false, "pid": 4015}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.689, "sample_count": 2152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22956, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:04:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T12:04:29+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-06T12:04:28+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T12:04:28.995809+09:00", "mtime_epoch": 1783307068.9958088, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218096}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:04:29+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1066.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.509, "pushed": 0.509, "read": 0.509}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.167, "count": 14, "max_ms": 7.505, "p95_ms": 7.505}, "gstreamer_push": {"avg_ms": 1.131, "count": 14, "max_ms": 1.346, "p95_ms": 1.346}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.338, "count": 14, "max_ms": 58.424, "p95_ms": 58.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.227, "count": 14, "max_ms": 106.093, "p95_ms": 106.093}, "yolo11_640_fire_11class": {"avg_ms": 58.962, "count": 14, "max_ms": 65.705, "p95_ms": 65.705}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.192, "count": 14, "max_ms": 105.593, "p95_ms": 105.593}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.836, "count": 14, "max_ms": 161.332, "p95_ms": 161.332}, "read_interval": {"avg_ms": 2386.148, "count": 14, "max_ms": 26006.235, "p95_ms": 26006.235}, "triton_wait": {"avg_ms": 68.589, "count": 14, "max_ms": 106.103, "p95_ms": 106.103}, "visualize": {"avg_ms": 4.429, "count": 14, "max_ms": 5.935, "p95_ms": 5.935}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.487, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4693.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:04:29+09:00", "last_error_summary": "[NULL @ 0x55db63165200] missing picture in access unit with size 18 [h264 @ 0x55db631f4700] no frame! [in#0/rtsp @ 0x55db63160e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55db63160e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55db63165840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T12:04:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.541, "last_started_at": "2026-07-06T12:02:15+09:00", "pid": 4015, "process": {"exists": false, "pid": 4015}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.689, "sample_count": 2152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22956, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:04:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T12:04:29+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T12:04:28+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T12:04:28.995809+09:00", "mtime_epoch": 1783307068.9958088, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218096}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:04:29+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1066.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.509, "pushed": 0.509, "read": 0.509}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.167, "count": 14, "max_ms": 7.505, "p95_ms": 7.505}, "gstreamer_push": {"avg_ms": 1.131, "count": 14, "max_ms": 1.346, "p95_ms": 1.346}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.338, "count": 14, "max_ms": 58.424, "p95_ms": 58.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.227, "count": 14, "max_ms": 106.093, "p95_ms": 106.093}, "yolo11_640_fire_11class": {"avg_ms": 58.962, "count": 14, "max_ms": 65.705, "p95_ms": 65.705}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.192, "count": 14, "max_ms": 105.593, "p95_ms": 105.593}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.836, "count": 14, "max_ms": 161.332, "p95_ms": 161.332}, "read_interval": {"avg_ms": 2386.148, "count": 14, "max_ms": 26006.235, "p95_ms": 26006.235}, "triton_wait": {"avg_ms": 68.589, "count": 14, "max_ms": 106.103, "p95_ms": 106.103}, "visualize": {"avg_ms": 4.429, "count": 14, "max_ms": 5.935, "p95_ms": 5.935}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.492, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4708.238, "limit_mb": 8796093022207.996}, "detail": {"pid": 4334, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:04:29+09:00", "last_error_summary": "[NULL @ 0x55db63165200] missing picture in access unit with size 18 [h264 @ 0x55db631f4700] no frame! [in#0/rtsp @ 0x55db63160e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55db63160e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55db63165840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T12:04:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:04:31+09:00", "pid": 4334, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.09, "name": "ffmpeg", "pid": 4334, "rss_mb": 58.09, "state": "S (sleeping)", "threads": 1}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.46, "sample_count": 2110, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22956, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:04:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 4.358, "last_frame_pushed_at": "2026-07-06T12:04:29+09:00", "last_frame_read_age_sec": 4.479, "last_frame_read_at": "2026-07-06T12:04:28+09:00", "mjpeg": {"age_sec": 4.343, "exists": true, "mtime": "2026-07-06T12:04:29.119811+09:00", "mtime_epoch": 1783307069.119811, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217617}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:04:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1066.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.228, "pushed": 2.228, "read": 2.228}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.291, "count": 13, "max_ms": 7.505, "p95_ms": 7.505}, "gstreamer_push": {"avg_ms": 1.126, "count": 13, "max_ms": 1.346, "p95_ms": 1.346}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.12, "count": 13, "max_ms": 58.424, "p95_ms": 58.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.941, "count": 13, "max_ms": 106.093, "p95_ms": 106.093}, "yolo11_640_fire_11class": {"avg_ms": 59.028, "count": 13, "max_ms": 65.705, "p95_ms": 65.705}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.465, "count": 13, "max_ms": 105.593, "p95_ms": 105.593}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.168, "count": 13, "max_ms": 161.332, "p95_ms": 161.332}, "read_interval": {"avg_ms": 2104.715, "count": 13, "max_ms": 26006.235, "p95_ms": 26006.235}, "triton_wait": {"avg_ms": 69.277, "count": 13, "max_ms": 106.103, "p95_ms": 106.103}, "visualize": {"avg_ms": 4.495, "count": 13, "max_ms": 5.935, "p95_ms": 5.935}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.834, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 120}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4803.961, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55de774bb080] error while decoding MB 31 22, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:06:06+09:00", "last_error_summary": "[h264 @ 0x55de774bb080] error while decoding MB 31 22, bytestream -13", "last_exit_at": "2026-07-06T12:03:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.227, "last_started_at": "2026-07-06T12:03:46+09:00", "pid": 4227, "process": {"exists": false, "pid": 4227}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 577.453, "sample_count": 571, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14423, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:06:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T12:06:06+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T12:06:06+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T12:06:06.685543+09:00", "mtime_epoch": 1783307166.6855433, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206300}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:06:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1195.652, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 57, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.901, "pushed": 1.901, "read": 1.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.081, "count": 56, "max_ms": 51.46, "p95_ms": 45.233}, "gstreamer_push": {"avg_ms": 0.95, "count": 57, "max_ms": 3.166, "p95_ms": 0.975}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.238, "count": 56, "max_ms": 78.234, "p95_ms": 70.121}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 91.479, "count": 56, "max_ms": 135.226, "p95_ms": 121.447}, "yolo11_640_fire_11class": {"avg_ms": 55.583, "count": 56, "max_ms": 87.002, "p95_ms": 74.599}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.924, "count": 56, "max_ms": 122.745, "p95_ms": 122.052}}, "truncated_count": 0}, "processing_total": {"avg_ms": 528.029, "count": 57, "max_ms": 647.388, "p95_ms": 609.035}, "read_interval": {"avg_ms": 535.088, "count": 57, "max_ms": 655.082, "p95_ms": 617.393}, "triton_wait": {"avg_ms": 92.916, "count": 56, "max_ms": 135.237, "p95_ms": 122.758}, "visualize": {"avg_ms": 22.06, "count": 56, "max_ms": 26.699, "p95_ms": 25.965}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 120}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4810.336, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:06:06+09:00", "last_error_summary": "[h264 @ 0x55de774bb080] error while decoding MB 31 22, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:06:06+09:00", "last_error_summary": "[h264 @ 0x55de774bb080] error while decoding MB 31 22, bytestream -13", "last_exit_at": "2026-07-06T12:06:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.705, "last_started_at": "2026-07-06T12:03:46+09:00", "pid": 4227, "process": {"exists": false, "pid": 4227}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 577.453, "sample_count": 571, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14424, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:06:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T12:06:07+09:00", "last_frame_read_age_sec": 0.531, "last_frame_read_at": "2026-07-06T12:06:06+09:00", "mjpeg": {"age_sec": 0.519, "exists": true, "mtime": "2026-07-06T12:06:06.685543+09:00", "mtime_epoch": 1783307166.6855433, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206300}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:06:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1203.496, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.907, "pushed": 1.907, "read": 1.874}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.102, "count": 56, "max_ms": 51.46, "p95_ms": 45.233}, "gstreamer_push": {"avg_ms": 0.969, "count": 57, "max_ms": 3.166, "p95_ms": 1.949}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.2, "count": 56, "max_ms": 78.234, "p95_ms": 70.121}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 91.169, "count": 56, "max_ms": 135.226, "p95_ms": 121.447}, "yolo11_640_fire_11class": {"avg_ms": 55.254, "count": 56, "max_ms": 87.002, "p95_ms": 74.599}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.072, "count": 56, "max_ms": 122.381, "p95_ms": 121.105}}, "truncated_count": 0}, "processing_total": {"avg_ms": 527.282, "count": 57, "max_ms": 647.388, "p95_ms": 609.035}, "read_interval": {"avg_ms": 534.341, "count": 56, "max_ms": 655.082, "p95_ms": 617.393}, "triton_wait": {"avg_ms": 92.199, "count": 56, "max_ms": 135.237, "p95_ms": 122.394}, "visualize": {"avg_ms": 22.181, "count": 56, "max_ms": 26.699, "p95_ms": 26.475}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 120}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4810.328, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:06:06+09:00", "last_error_summary": "[h264 @ 0x55de774bb080] error while decoding MB 31 22, bytestream -13", "last_exit_at": "2026-07-06T12:06:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.708, "last_started_at": "2026-07-06T12:03:46+09:00", "pid": 4227, "process": {"exists": false, "pid": 4227}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 577.453, "sample_count": 571, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14424, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:06:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T12:06:07+09:00", "last_frame_read_age_sec": 0.534, "last_frame_read_at": "2026-07-06T12:06:06+09:00", "mjpeg": {"age_sec": 0.522, "exists": true, "mtime": "2026-07-06T12:06:06.685543+09:00", "mtime_epoch": 1783307166.6855433, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206300}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:06:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1203.496, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.907, "pushed": 1.907, "read": 1.873}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.102, "count": 56, "max_ms": 51.46, "p95_ms": 45.233}, "gstreamer_push": {"avg_ms": 0.969, "count": 57, "max_ms": 3.166, "p95_ms": 1.949}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.2, "count": 56, "max_ms": 78.234, "p95_ms": 70.121}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 91.169, "count": 56, "max_ms": 135.226, "p95_ms": 121.447}, "yolo11_640_fire_11class": {"avg_ms": 55.254, "count": 56, "max_ms": 87.002, "p95_ms": 74.599}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.072, "count": 56, "max_ms": 122.381, "p95_ms": 121.105}}, "truncated_count": 0}, "processing_total": {"avg_ms": 527.282, "count": 57, "max_ms": 647.388, "p95_ms": 609.035}, "read_interval": {"avg_ms": 534.341, "count": 56, "max_ms": 655.082, "p95_ms": 617.393}, "triton_wait": {"avg_ms": 92.199, "count": 56, "max_ms": 135.237, "p95_ms": 122.394}, "visualize": {"avg_ms": 22.181, "count": 56, "max_ms": 26.699, "p95_ms": 26.475}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 120}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4919.84, "limit_mb": 8796093022207.996}, "detail": {"pid": 4546, "restart_count": 17, "start_count": 18}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:06:06+09:00", "last_error_summary": "[h264 @ 0x55de774bb080] error while decoding MB 31 22, bytestream -13", "last_exit_at": "2026-07-06T12:06:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:06:09+09:00", "pid": 4546, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.906, "name": "ffmpeg", "pid": 4546, "rss_mb": 139.906, "state": "S (sleeping)", "threads": 106}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 577.453, "sample_count": 561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14424, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:06:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.486, "last_frame_pushed_at": "2026-07-06T12:06:07+09:00", "last_frame_read_age_sec": 5.015, "last_frame_read_at": "2026-07-06T12:06:06+09:00", "mjpeg": {"age_sec": 4.471, "exists": true, "mtime": "2026-07-06T12:06:07.217553+09:00", "mtime_epoch": 1783307167.2175527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206001}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:06:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1203.496, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.655, "pushed": 1.655, "read": 1.622}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.13, "count": 48, "max_ms": 49.646, "p95_ms": 41.461}, "gstreamer_push": {"avg_ms": 0.982, "count": 49, "max_ms": 3.166, "p95_ms": 1.949}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.892, "count": 48, "max_ms": 78.234, "p95_ms": 63.103}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 89.854, "count": 48, "max_ms": 135.226, "p95_ms": 120.61}, "yolo11_640_fire_11class": {"avg_ms": 54.522, "count": 48, "max_ms": 87.002, "p95_ms": 74.599}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.056, "count": 48, "max_ms": 121.105, "p95_ms": 112.688}}, "truncated_count": 0}, "processing_total": {"avg_ms": 517.439, "count": 49, "max_ms": 647.388, "p95_ms": 582.802}, "read_interval": {"avg_ms": 524.241, "count": 48, "max_ms": 655.082, "p95_ms": 590.745}, "triton_wait": {"avg_ms": 90.916, "count": 48, "max_ms": 135.237, "p95_ms": 120.623}, "visualize": {"avg_ms": 22.226, "count": 48, "max_ms": 26.699, "p95_ms": 26.475}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.601, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 31, "tracking_summary": 122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4934.059, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556568451240] error while decoding MB 2 16, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:08:26+09:00", "last_error_summary": "[h264 @ 0x556568451240] error while decoding MB 2 16, bytestream -5", "last_exit_at": "2026-07-06T12:06:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.995, "last_started_at": "2026-07-06T12:06:09+09:00", "pid": 4546, "process": {"exists": false, "pid": 4546}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 569.274, "sample_count": 588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14710, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:08:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T12:08:26+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T12:08:26+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T12:08:26.643835+09:00", "mtime_epoch": 1783307306.6438353, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207768}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:08:26+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.902, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 10.781, "recorders": [{"age_sec": 10.781, "frame_count": 70, "is_recording": true, "is_stopping": false, "pid": 4653, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.75, "name": "ffmpeg", "pid": 4653, "rss_mb": 185.75, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 30, "returncode": null, "running": true, "started_at": "2026-07-06T12:08:15+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_120815_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [4653], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 62, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.071, "pushed": 2.071, "read": 2.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.241, "count": 61, "max_ms": 44.382, "p95_ms": 32.533}, "gstreamer_push": {"avg_ms": 0.874, "count": 62, "max_ms": 1.48, "p95_ms": 0.94}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.704, "count": 61, "max_ms": 89.531, "p95_ms": 72.008}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 94.852, "count": 61, "max_ms": 128.185, "p95_ms": 123.226}, "yolo11_640_fire_11class": {"avg_ms": 59.011, "count": 61, "max_ms": 88.799, "p95_ms": 81.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 89.762, "count": 61, "max_ms": 130.006, "p95_ms": 117.808}}, "truncated_count": 0}, "processing_total": {"avg_ms": 482.637, "count": 62, "max_ms": 941.317, "p95_ms": 562.481}, "read_interval": {"avg_ms": 489.382, "count": 62, "max_ms": 949.085, "p95_ms": 572.99}, "triton_wait": {"avg_ms": 95.731, "count": 61, "max_ms": 130.021, "p95_ms": 125.228}, "visualize": {"avg_ms": 19.702, "count": 61, "max_ms": 26.736, "p95_ms": 23.857}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 31, "tracking_summary": 122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4932.801, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:08:26+09:00", "last_error_summary": "[h264 @ 0x556568451240] error while decoding MB 2 16, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:08:26+09:00", "last_error_summary": "[h264 @ 0x556568451240] error while decoding MB 2 16, bytestream -5", "last_exit_at": "2026-07-06T12:08:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.501, "last_started_at": "2026-07-06T12:06:09+09:00", "pid": 4546, "process": {"exists": false, "pid": 4546}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 569.274, "sample_count": 588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14711, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:08:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-06T12:08:27+09:00", "last_frame_read_age_sec": 0.557, "last_frame_read_at": "2026-07-06T12:08:26+09:00", "mjpeg": {"age_sec": 0.545, "exists": true, "mtime": "2026-07-06T12:08:26.643835+09:00", "mtime_epoch": 1783307306.6438353, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207768}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:08:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.902, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 11.286, "recorders": [{"age_sec": 11.286, "frame_count": 71, "is_recording": true, "is_stopping": false, "pid": 4653, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.75, "name": "ffmpeg", "pid": 4653, "rss_mb": 185.75, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 29, "returncode": null, "running": true, "started_at": "2026-07-06T12:08:15+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_120815_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [4653], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.069, "pushed": 2.069, "read": 2.036}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.42, "count": 61, "max_ms": 44.382, "p95_ms": 33.279}, "gstreamer_push": {"avg_ms": 0.874, "count": 62, "max_ms": 1.48, "p95_ms": 0.94}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.135, "count": 61, "max_ms": 89.531, "p95_ms": 74.301}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 95.25, "count": 61, "max_ms": 128.185, "p95_ms": 123.226}, "yolo11_640_fire_11class": {"avg_ms": 59.393, "count": 61, "max_ms": 88.799, "p95_ms": 81.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 90.166, "count": 61, "max_ms": 130.006, "p95_ms": 117.808}}, "truncated_count": 0}, "processing_total": {"avg_ms": 484.225, "count": 62, "max_ms": 941.317, "p95_ms": 562.481}, "read_interval": {"avg_ms": 489.859, "count": 61, "max_ms": 949.085, "p95_ms": 572.99}, "triton_wait": {"avg_ms": 96.13, "count": 61, "max_ms": 130.021, "p95_ms": 125.228}, "visualize": {"avg_ms": 19.728, "count": 61, "max_ms": 26.736, "p95_ms": 23.857}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 31, "tracking_summary": 122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4932.914, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:08:26+09:00", "last_error_summary": "[h264 @ 0x556568451240] error while decoding MB 2 16, bytestream -5", "last_exit_at": "2026-07-06T12:08:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.505, "last_started_at": "2026-07-06T12:06:09+09:00", "pid": 4546, "process": {"exists": false, "pid": 4546}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 569.274, "sample_count": 588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14711, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:08:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T12:08:27+09:00", "last_frame_read_age_sec": 0.56, "last_frame_read_at": "2026-07-06T12:08:26+09:00", "mjpeg": {"age_sec": 0.548, "exists": true, "mtime": "2026-07-06T12:08:26.643835+09:00", "mtime_epoch": 1783307306.6438353, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207768}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:08:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.902, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 11.289, "recorders": [{"age_sec": 11.289, "frame_count": 71, "is_recording": true, "is_stopping": false, "pid": 4653, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.75, "name": "ffmpeg", "pid": 4653, "rss_mb": 185.75, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 29, "returncode": null, "running": true, "started_at": "2026-07-06T12:08:15+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_120815_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [4653], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.069, "pushed": 2.069, "read": 2.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.42, "count": 61, "max_ms": 44.382, "p95_ms": 33.279}, "gstreamer_push": {"avg_ms": 0.874, "count": 62, "max_ms": 1.48, "p95_ms": 0.94}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.135, "count": 61, "max_ms": 89.531, "p95_ms": 74.301}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 95.25, "count": 61, "max_ms": 128.185, "p95_ms": 123.226}, "yolo11_640_fire_11class": {"avg_ms": 59.393, "count": 61, "max_ms": 88.799, "p95_ms": 81.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 90.166, "count": 61, "max_ms": 130.006, "p95_ms": 117.808}}, "truncated_count": 0}, "processing_total": {"avg_ms": 484.225, "count": 62, "max_ms": 941.317, "p95_ms": 562.481}, "read_interval": {"avg_ms": 489.859, "count": 61, "max_ms": 949.085, "p95_ms": 572.99}, "triton_wait": {"avg_ms": 96.13, "count": 61, "max_ms": 130.021, "p95_ms": 125.228}, "visualize": {"avg_ms": 19.728, "count": 61, "max_ms": 26.736, "p95_ms": 23.857}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 31, "tracking_summary": 122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5045.125, "limit_mb": 8796093022207.996}, "detail": {"pid": 4713, "restart_count": 18, "start_count": 19}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:08:26+09:00", "last_error_summary": "[h264 @ 0x556568451240] error while decoding MB 2 16, bytestream -5", "last_exit_at": "2026-07-06T12:08:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:08:29+09:00", "pid": 4713, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.375, "name": "ffmpeg", "pid": 4713, "rss_mb": 141.375, "state": "S (sleeping)", "threads": 106}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 569.274, "sample_count": 579, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14711, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:08:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.455, "last_frame_pushed_at": "2026-07-06T12:08:27+09:00", "last_frame_read_age_sec": 5.009, "last_frame_read_at": "2026-07-06T12:08:26+09:00", "mjpeg": {"age_sec": 4.438, "exists": true, "mtime": "2026-07-06T12:08:27.203837+09:00", "mtime_epoch": 1783307307.2038374, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204275}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:08:31+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1161.902, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 15.74, "recorders": [{"age_sec": 15.74, "frame_count": 71, "is_recording": true, "is_stopping": false, "pid": 4653, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.75, "name": "ffmpeg", "pid": 4653, "rss_mb": 185.75, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 29, "returncode": null, "running": true, "started_at": "2026-07-06T12:08:15+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260706_120815_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [4653], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.745, "pushed": 1.745, "read": 1.711}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.714, "count": 51, "max_ms": 44.382, "p95_ms": 34.859}, "gstreamer_push": {"avg_ms": 0.874, "count": 52, "max_ms": 1.48, "p95_ms": 0.94}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.157, "count": 51, "max_ms": 89.531, "p95_ms": 74.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 97.11, "count": 51, "max_ms": 128.185, "p95_ms": 125.218}, "yolo11_640_fire_11class": {"avg_ms": 60.636, "count": 51, "max_ms": 88.799, "p95_ms": 85.158}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 91.687, "count": 51, "max_ms": 130.006, "p95_ms": 118.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 489.588, "count": 52, "max_ms": 941.317, "p95_ms": 569.918}, "read_interval": {"avg_ms": 495.162, "count": 51, "max_ms": 949.085, "p95_ms": 577.537}, "triton_wait": {"avg_ms": 98.143, "count": 51, "max_ms": 130.021, "p95_ms": 127.874}, "visualize": {"avg_ms": 19.9, "count": 51, "max_ms": 26.736, "p95_ms": 24.234}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.804, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 31, "tracking_summary": 124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4778.84, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:10:41+09:00", "last_error_summary": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "last_exit_at": "2026-07-06T12:08:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.958, "last_started_at": "2026-07-06T12:08:29+09:00", "pid": 4713, "process": {"exists": false, "pid": 4713}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 532.224, "sample_count": 614, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14989, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:10:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.046, "last_frame_pushed_at": "2026-07-06T12:10:41+09:00", "last_frame_read_age_sec": 0.042, "last_frame_read_at": "2026-07-06T12:10:41+09:00", "mjpeg": {"age_sec": 0.03, "exists": true, "mtime": "2026-07-06T12:10:41.568544+09:00", "mtime_epoch": 1783307441.5685441, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205066}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:10:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1162.16, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 65, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.178, "pushed": 2.178, "read": 2.178}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.784, "count": 64, "max_ms": 32.275, "p95_ms": 28.358}, "gstreamer_push": {"avg_ms": 0.862, "count": 65, "max_ms": 0.947, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.921, "count": 64, "max_ms": 75.481, "p95_ms": 72.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 97.152, "count": 64, "max_ms": 131.104, "p95_ms": 124.087}, "yolo11_640_fire_11class": {"avg_ms": 58.578, "count": 64, "max_ms": 88.58, "p95_ms": 75.796}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.037, "count": 64, "max_ms": 131.118, "p95_ms": 117.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 459.329, "count": 65, "max_ms": 496.601, "p95_ms": 491.01}, "read_interval": {"avg_ms": 465.754, "count": 65, "max_ms": 502.38, "p95_ms": 497.753}, "triton_wait": {"avg_ms": 98.243, "count": 64, "max_ms": 131.128, "p95_ms": 125.574}, "visualize": {"avg_ms": 17.729, "count": 64, "max_ms": 20.95, "p95_ms": 20.43}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 31, "tracking_summary": 124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4780.879, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:10:41+09:00", "last_error_summary": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:10:41+09:00", "last_error_summary": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "last_exit_at": "2026-07-06T12:10:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.43, "last_started_at": "2026-07-06T12:08:29+09:00", "pid": 4713, "process": {"exists": false, "pid": 4713}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 532.224, "sample_count": 615, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14990, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:10:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T12:10:42+09:00", "last_frame_read_age_sec": 0.513, "last_frame_read_at": "2026-07-06T12:10:41+09:00", "mjpeg": {"age_sec": 0.502, "exists": true, "mtime": "2026-07-06T12:10:41.568544+09:00", "mtime_epoch": 1783307441.5685441, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205066}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:10:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1162.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.176, "pushed": 2.176, "read": 2.143}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.806, "count": 64, "max_ms": 32.275, "p95_ms": 28.358}, "gstreamer_push": {"avg_ms": 0.862, "count": 65, "max_ms": 0.947, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.508, "count": 64, "max_ms": 89.405, "p95_ms": 72.663}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 97.926, "count": 64, "max_ms": 137.476, "p95_ms": 125.995}, "yolo11_640_fire_11class": {"avg_ms": 59.037, "count": 64, "max_ms": 88.58, "p95_ms": 80.035}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.765, "count": 64, "max_ms": 132.862, "p95_ms": 120.677}}, "truncated_count": 0}, "processing_total": {"avg_ms": 459.928, "count": 65, "max_ms": 509.165, "p95_ms": 493.039}, "read_interval": {"avg_ms": 465.557, "count": 64, "max_ms": 502.38, "p95_ms": 497.753}, "triton_wait": {"avg_ms": 99.017, "count": 64, "max_ms": 137.489, "p95_ms": 126.009}, "visualize": {"avg_ms": 17.718, "count": 64, "max_ms": 20.95, "p95_ms": 20.43}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 31, "tracking_summary": 124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4780.871, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:10:41+09:00", "last_error_summary": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "last_exit_at": "2026-07-06T12:10:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.433, "last_started_at": "2026-07-06T12:08:29+09:00", "pid": 4713, "process": {"exists": false, "pid": 4713}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 532.224, "sample_count": 615, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14990, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:10:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T12:10:42+09:00", "last_frame_read_age_sec": 0.516, "last_frame_read_at": "2026-07-06T12:10:41+09:00", "mjpeg": {"age_sec": 0.505, "exists": true, "mtime": "2026-07-06T12:10:41.568544+09:00", "mtime_epoch": 1783307441.5685441, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205066}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:10:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1162.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.176, "pushed": 2.176, "read": 2.142}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.806, "count": 64, "max_ms": 32.275, "p95_ms": 28.358}, "gstreamer_push": {"avg_ms": 0.862, "count": 65, "max_ms": 0.947, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.508, "count": 64, "max_ms": 89.405, "p95_ms": 72.663}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 97.926, "count": 64, "max_ms": 137.476, "p95_ms": 125.995}, "yolo11_640_fire_11class": {"avg_ms": 59.037, "count": 64, "max_ms": 88.58, "p95_ms": 80.035}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.765, "count": 64, "max_ms": 132.862, "p95_ms": 120.677}}, "truncated_count": 0}, "processing_total": {"avg_ms": 459.928, "count": 65, "max_ms": 509.165, "p95_ms": 493.039}, "read_interval": {"avg_ms": 465.557, "count": 64, "max_ms": 502.38, "p95_ms": 497.753}, "triton_wait": {"avg_ms": 99.017, "count": 64, "max_ms": 137.489, "p95_ms": 126.009}, "visualize": {"avg_ms": 17.718, "count": 64, "max_ms": 20.95, "p95_ms": 20.43}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 31, "tracking_summary": 124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4887.754, "limit_mb": 8796093022207.996}, "detail": {"pid": 4820, "restart_count": 19, "start_count": 20}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:10:41+09:00", "last_error_summary": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "last_exit_at": "2026-07-06T12:10:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:10:44+09:00", "pid": 4820, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.5, "name": "ffmpeg", "pid": 4820, "rss_mb": 139.5, "state": "S (sleeping)", "threads": 106}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 528.888, "sample_count": 607, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14990, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:10:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.427, "last_frame_pushed_at": "2026-07-06T12:10:42+09:00", "last_frame_read_age_sec": 4.937, "last_frame_read_at": "2026-07-06T12:10:41+09:00", "mjpeg": {"age_sec": 4.41, "exists": true, "mtime": "2026-07-06T12:10:42.084548+09:00", "mtime_epoch": 1783307442.0845475, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:10:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1691.559, "name": "celery", "pid": 237, "rss_mb": 1162.16, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.849, "pushed": 1.849, "read": 1.815}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.173, "count": 54, "max_ms": 32.275, "p95_ms": 28.623}, "gstreamer_push": {"avg_ms": 0.857, "count": 55, "max_ms": 0.926, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.572, "count": 54, "max_ms": 89.405, "p95_ms": 72.685}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 97.602, "count": 54, "max_ms": 137.476, "p95_ms": 124.087}, "yolo11_640_fire_11class": {"avg_ms": 58.286, "count": 54, "max_ms": 88.444, "p95_ms": 75.796}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.135, "count": 54, "max_ms": 132.862, "p95_ms": 120.677}}, "truncated_count": 0}, "processing_total": {"avg_ms": 462.079, "count": 55, "max_ms": 509.165, "p95_ms": 495.856}, "read_interval": {"avg_ms": 467.62, "count": 54, "max_ms": 502.38, "p95_ms": 498.761}, "triton_wait": {"avg_ms": 98.533, "count": 54, "max_ms": 137.489, "p95_ms": 125.574}, "visualize": {"avg_ms": 17.929, "count": 54, "max_ms": 20.95, "p95_ms": 20.47}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.751, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 45, "tracking_summary": 127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5235.832, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:10:41+09:00", "last_error_summary": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:10:41+09:00", "last_error_summary": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "last_exit_at": "2026-07-06T12:13:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.848, "last_started_at": "2026-07-06T12:10:44+09:00", "pid": 4820, "process": {"exists": false, "pid": 4820}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 825.746, "sample_count": 536, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:13:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T12:13:29+09:00", "last_frame_read_age_sec": 0.693, "last_frame_read_at": "2026-07-06T12:13:28+09:00", "mjpeg": {"age_sec": 0.68, "exists": true, "mtime": "2026-07-06T12:13:28.661887+09:00", "mtime_epoch": 1783307608.661887, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209652}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:13:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1590.547, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.353, "pushed": 1.353, "read": 1.319}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 45.535, "count": 39, "max_ms": 65.08, "p95_ms": 59.201}, "gstreamer_push": {"avg_ms": 0.901, "count": 40, "max_ms": 0.951, "p95_ms": 0.949}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.781, "count": 39, "max_ms": 80.443, "p95_ms": 78.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 98.988, "count": 39, "max_ms": 129.299, "p95_ms": 127.575}, "yolo11_640_fire_11class": {"avg_ms": 57.34, "count": 39, "max_ms": 79.499, "p95_ms": 74.914}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 93.85, "count": 39, "max_ms": 129.311, "p95_ms": 120.325}}, "truncated_count": 0}, "processing_total": {"avg_ms": 750.144, "count": 40, "max_ms": 871.479, "p95_ms": 822.694}, "read_interval": {"avg_ms": 758.478, "count": 39, "max_ms": 878.561, "p95_ms": 832.79}, "triton_wait": {"avg_ms": 99.409, "count": 39, "max_ms": 129.321, "p95_ms": 127.589}, "visualize": {"avg_ms": 26.503, "count": 39, "max_ms": 34.094, "p95_ms": 33.429}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.559, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 45, "tracking_summary": 127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5236.004, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:10:41+09:00", "last_error_summary": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "last_exit_at": "2026-07-06T12:13:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.851, "last_started_at": "2026-07-06T12:10:44+09:00", "pid": 4820, "process": {"exists": false, "pid": 4820}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 825.746, "sample_count": 536, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:13:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T12:13:29+09:00", "last_frame_read_age_sec": 0.696, "last_frame_read_at": "2026-07-06T12:13:28+09:00", "mjpeg": {"age_sec": 0.683, "exists": true, "mtime": "2026-07-06T12:13:28.661887+09:00", "mtime_epoch": 1783307608.661887, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209652}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:13:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1590.547, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.353, "pushed": 1.353, "read": 1.319}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 45.535, "count": 39, "max_ms": 65.08, "p95_ms": 59.201}, "gstreamer_push": {"avg_ms": 0.901, "count": 40, "max_ms": 0.951, "p95_ms": 0.949}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.781, "count": 39, "max_ms": 80.443, "p95_ms": 78.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 98.988, "count": 39, "max_ms": 129.299, "p95_ms": 127.575}, "yolo11_640_fire_11class": {"avg_ms": 57.34, "count": 39, "max_ms": 79.499, "p95_ms": 74.914}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 93.85, "count": 39, "max_ms": 129.311, "p95_ms": 120.325}}, "truncated_count": 0}, "processing_total": {"avg_ms": 750.144, "count": 40, "max_ms": 871.479, "p95_ms": 822.694}, "read_interval": {"avg_ms": 758.478, "count": 39, "max_ms": 878.561, "p95_ms": 832.79}, "triton_wait": {"avg_ms": 99.409, "count": 39, "max_ms": 129.321, "p95_ms": 127.589}, "visualize": {"avg_ms": 26.503, "count": 39, "max_ms": 34.094, "p95_ms": 33.429}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.562, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 45, "tracking_summary": 127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5338.098, "limit_mb": 8796093022207.996}, "detail": {"pid": 4987, "restart_count": 20, "start_count": 21}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:10:41+09:00", "last_error_summary": "[h264 @ 0x563eb7d52bc0] error while decoding MB 3 37, bytestream -9", "last_exit_at": "2026-07-06T12:13:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:13:31+09:00", "pid": 4987, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.5, "name": "ffmpeg", "pid": 4987, "rss_mb": 139.5, "state": "S (sleeping)", "threads": 106}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 825.746, "sample_count": 532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:13:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.658, "last_frame_pushed_at": "2026-07-06T12:13:29+09:00", "last_frame_read_age_sec": 5.347, "last_frame_read_at": "2026-07-06T12:13:28+09:00", "mjpeg": {"age_sec": 4.637, "exists": true, "mtime": "2026-07-06T12:13:29.357893+09:00", "mtime_epoch": 1783307609.3578935, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209905}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:13:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1590.547, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.154, "pushed": 1.154, "read": 1.12}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 45.576, "count": 33, "max_ms": 65.08, "p95_ms": 59.201}, "gstreamer_push": {"avg_ms": 0.902, "count": 34, "max_ms": 0.951, "p95_ms": 0.95}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.239, "count": 33, "max_ms": 80.443, "p95_ms": 78.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 100.096, "count": 33, "max_ms": 129.299, "p95_ms": 127.575}, "yolo11_640_fire_11class": {"avg_ms": 57.586, "count": 33, "max_ms": 79.499, "p95_ms": 74.914}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 94.968, "count": 33, "max_ms": 129.311, "p95_ms": 120.325}}, "truncated_count": 0}, "processing_total": {"avg_ms": 745.681, "count": 34, "max_ms": 823.026, "p95_ms": 822.694}, "read_interval": {"avg_ms": 754.241, "count": 33, "max_ms": 832.79, "p95_ms": 829.625}, "triton_wait": {"avg_ms": 100.591, "count": 33, "max_ms": 129.321, "p95_ms": 127.589}, "visualize": {"avg_ms": 26.627, "count": 33, "max_ms": 34.094, "p95_ms": 33.429}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.46, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 48, "tracking_summary": 129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4815.297, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5562fd424480] error while decoding MB 9 23, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:15:47+09:00", "last_error_summary": "[h264 @ 0x5562fd424480] error while decoding MB 9 23, bytestream -15", "last_exit_at": "2026-07-06T12:13:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.151, "last_started_at": "2026-07-06T12:13:31+09:00", "pid": 4987, "process": {"exists": false, "pid": 4987}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 831.546, "sample_count": 515, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:15:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-06T12:15:47+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T12:15:47+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-06T12:15:47.111292+09:00", "mtime_epoch": 1783307747.1112921, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208633}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:15:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1163.801, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.123, "pushed": 2.123, "read": 2.123}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.546, "count": 62, "max_ms": 33.978, "p95_ms": 30.641}, "gstreamer_push": {"avg_ms": 0.903, "count": 63, "max_ms": 2.039, "p95_ms": 0.981}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.776, "count": 62, "max_ms": 82.83, "p95_ms": 79.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 94.383, "count": 62, "max_ms": 125.707, "p95_ms": 116.961}, "yolo11_640_fire_11class": {"avg_ms": 61.827, "count": 62, "max_ms": 86.094, "p95_ms": 84.095}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 91.426, "count": 62, "max_ms": 132.431, "p95_ms": 121.418}}, "truncated_count": 0}, "processing_total": {"avg_ms": 474.029, "count": 63, "max_ms": 683.3, "p95_ms": 555.8}, "read_interval": {"avg_ms": 480.789, "count": 63, "max_ms": 689.613, "p95_ms": 563.337}, "triton_wait": {"avg_ms": 96.791, "count": 62, "max_ms": 132.445, "p95_ms": 124.486}, "visualize": {"avg_ms": 20.106, "count": 62, "max_ms": 26.231, "p95_ms": 23.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.681, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 48, "tracking_summary": 129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4815.797, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:15:47+09:00", "last_error_summary": "[h264 @ 0x5562fd424480] error while decoding MB 9 23, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:15:47+09:00", "last_error_summary": "[h264 @ 0x5562fd424480] error while decoding MB 9 23, bytestream -15", "last_exit_at": "2026-07-06T12:15:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.59, "last_started_at": "2026-07-06T12:13:31+09:00", "pid": 4987, "process": {"exists": false, "pid": 4987}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 831.546, "sample_count": 515, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:15:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T12:15:47+09:00", "last_frame_read_age_sec": 0.49, "last_frame_read_at": "2026-07-06T12:15:47+09:00", "mjpeg": {"age_sec": 0.473, "exists": true, "mtime": "2026-07-06T12:15:47.111292+09:00", "mtime_epoch": 1783307747.1112921, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208633}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:15:47+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1163.801, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.127, "pushed": 2.127, "read": 2.094}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.512, "count": 63, "max_ms": 33.978, "p95_ms": 30.641}, "gstreamer_push": {"avg_ms": 0.885, "count": 63, "max_ms": 1.93, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.78, "count": 63, "max_ms": 82.83, "p95_ms": 79.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 94.527, "count": 63, "max_ms": 125.707, "p95_ms": 116.961}, "yolo11_640_fire_11class": {"avg_ms": 61.94, "count": 63, "max_ms": 86.094, "p95_ms": 84.095}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 91.661, "count": 63, "max_ms": 132.431, "p95_ms": 121.418}}, "truncated_count": 0}, "processing_total": {"avg_ms": 471.266, "count": 63, "max_ms": 683.3, "p95_ms": 544.51}, "read_interval": {"avg_ms": 477.797, "count": 62, "max_ms": 689.613, "p95_ms": 552.452}, "triton_wait": {"avg_ms": 96.941, "count": 63, "max_ms": 132.445, "p95_ms": 124.486}, "visualize": {"avg_ms": 20.072, "count": 63, "max_ms": 26.231, "p95_ms": 23.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.614, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 48, "tracking_summary": 129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4815.789, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:15:47+09:00", "last_error_summary": "[h264 @ 0x5562fd424480] error while decoding MB 9 23, bytestream -15", "last_exit_at": "2026-07-06T12:15:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.593, "last_started_at": "2026-07-06T12:13:31+09:00", "pid": 4987, "process": {"exists": false, "pid": 4987}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 831.546, "sample_count": 515, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:15:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T12:15:47+09:00", "last_frame_read_age_sec": 0.493, "last_frame_read_at": "2026-07-06T12:15:47+09:00", "mjpeg": {"age_sec": 0.476, "exists": true, "mtime": "2026-07-06T12:15:47.111292+09:00", "mtime_epoch": 1783307747.1112921, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208633}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:15:47+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1163.801, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.127, "pushed": 2.127, "read": 2.093}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.512, "count": 63, "max_ms": 33.978, "p95_ms": 30.641}, "gstreamer_push": {"avg_ms": 0.885, "count": 63, "max_ms": 1.93, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.78, "count": 63, "max_ms": 82.83, "p95_ms": 79.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 94.527, "count": 63, "max_ms": 125.707, "p95_ms": 116.961}, "yolo11_640_fire_11class": {"avg_ms": 61.94, "count": 63, "max_ms": 86.094, "p95_ms": 84.095}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 91.661, "count": 63, "max_ms": 132.431, "p95_ms": 121.418}}, "truncated_count": 0}, "processing_total": {"avg_ms": 471.266, "count": 63, "max_ms": 683.3, "p95_ms": 544.51}, "read_interval": {"avg_ms": 477.797, "count": 62, "max_ms": 689.613, "p95_ms": 552.452}, "triton_wait": {"avg_ms": 96.941, "count": 63, "max_ms": 132.445, "p95_ms": 124.486}, "visualize": {"avg_ms": 20.072, "count": 63, "max_ms": 26.231, "p95_ms": 23.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.617, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 48, "tracking_summary": 129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4920.27, "limit_mb": 8796093022207.996}, "detail": {"pid": 5154, "restart_count": 21, "start_count": 22}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:15:47+09:00", "last_error_summary": "[h264 @ 0x5562fd424480] error while decoding MB 9 23, bytestream -15", "last_exit_at": "2026-07-06T12:15:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:15:50+09:00", "pid": 5154, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.785, "name": "ffmpeg", "pid": 5154, "rss_mb": 139.785, "state": "S (sleeping)", "threads": 106}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 836.458, "sample_count": 506, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:15:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.705, "last_frame_pushed_at": "2026-07-06T12:15:47+09:00", "last_frame_read_age_sec": 5.192, "last_frame_read_at": "2026-07-06T12:15:47+09:00", "mjpeg": {"age_sec": 4.688, "exists": true, "mtime": "2026-07-06T12:15:47.599298+09:00", "mtime_epoch": 1783307747.5992975, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:15:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1163.801, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 53, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.821, "pushed": 1.821, "read": 1.787}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.204, "count": 53, "max_ms": 33.978, "p95_ms": 30.641}, "gstreamer_push": {"avg_ms": 0.867, "count": 54, "max_ms": 0.997, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.141, "count": 53, "max_ms": 82.168, "p95_ms": 77.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 95.574, "count": 53, "max_ms": 125.707, "p95_ms": 117.295}, "yolo11_640_fire_11class": {"avg_ms": 61.214, "count": 53, "max_ms": 86.094, "p95_ms": 81.634}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 90.696, "count": 53, "max_ms": 132.431, "p95_ms": 117.998}}, "truncated_count": 0}, "processing_total": {"avg_ms": 465.745, "count": 54, "max_ms": 607.426, "p95_ms": 544.51}, "read_interval": {"avg_ms": 472.149, "count": 53, "max_ms": 613.178, "p95_ms": 552.452}, "triton_wait": {"avg_ms": 96.95, "count": 53, "max_ms": 132.445, "p95_ms": 121.431}, "visualize": {"avg_ms": 20.259, "count": 53, "max_ms": 24.1, "p95_ms": 23.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.66, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4743.164, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[NULL @ 0x55d650d0e200] missing picture in access unit with size 18 [h264 @ 0x55d650d8ab40] no frame! [vist#0:0/h264 @ 0x55d650d0e840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:16:40+09:00", "last_error_summary": "[NULL @ 0x55d650d0e200] missing picture in access unit with size 18 [h264 @ 0x55d650d8ab40] no frame! [vist#0:0/h264 @ 0x55d650d0e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T12:04:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 729.433, "last_started_at": "2026-07-06T12:04:31+09:00", "pid": 4334, "process": {"exists": false, "pid": 4334}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.696, "sample_count": 2233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T12:16:40+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T12:16:40+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-06T12:16:40.831894+09:00", "mtime_epoch": 1783307800.8318937, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236823}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:16:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1071.02, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 222, "pushed_frames": 222, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.426, "pushed": 7.426, "read": 7.426}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.309, "count": 222, "max_ms": 9.01, "p95_ms": 7.953}, "gstreamer_push": {"avg_ms": 1.126, "count": 222, "max_ms": 3.702, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.455, "count": 222, "max_ms": 82.878, "p95_ms": 66.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.186, "count": 222, "max_ms": 138.296, "p95_ms": 106.31}, "yolo11_640_fire_11class": {"avg_ms": 60.273, "count": 222, "max_ms": 91.47, "p95_ms": 78.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.503, "count": 222, "max_ms": 122.887, "p95_ms": 106.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.121, "count": 222, "max_ms": 193.539, "p95_ms": 163.336}, "read_interval": {"avg_ms": 134.903, "count": 222, "max_ms": 204.237, "p95_ms": 174.759}, "triton_wait": {"avg_ms": 72.17, "count": 222, "max_ms": 138.308, "p95_ms": 109.695}, "visualize": {"avg_ms": 5.111, "count": 222, "max_ms": 6.925, "p95_ms": 6.018}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4742.422, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:16:40+09:00", "last_error_summary": "[NULL @ 0x55d650d0e200] missing picture in access unit with size 18 [h264 @ 0x55d650d8ab40] no frame! [vist#0:0/h264 @ 0x55d650d0e840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:16:40+09:00", "last_error_summary": "[NULL @ 0x55d650d0e200] missing picture in access unit with size 18 [h264 @ 0x55d650d8ab40] no frame! [vist#0:0/h264 @ 0x55d650d0e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T12:16:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 729.509, "last_started_at": "2026-07-06T12:04:31+09:00", "pid": 4334, "process": {"exists": false, "pid": 4334}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.696, "sample_count": 2233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28457, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T12:16:40+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T12:16:40+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-06T12:16:40.831894+09:00", "mtime_epoch": 1783307800.8318937, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236823}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:16:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1071.02, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.441, "pushed": 7.441, "read": 7.407}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.296, "count": 222, "max_ms": 9.01, "p95_ms": 7.953}, "gstreamer_push": {"avg_ms": 1.126, "count": 223, "max_ms": 3.702, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.465, "count": 222, "max_ms": 82.878, "p95_ms": 66.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.374, "count": 222, "max_ms": 138.296, "p95_ms": 106.393}, "yolo11_640_fire_11class": {"avg_ms": 60.395, "count": 222, "max_ms": 91.47, "p95_ms": 78.831}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.752, "count": 222, "max_ms": 122.887, "p95_ms": 107.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.156, "count": 223, "max_ms": 193.539, "p95_ms": 163.336}, "read_interval": {"avg_ms": 134.903, "count": 222, "max_ms": 204.237, "p95_ms": 174.759}, "triton_wait": {"avg_ms": 72.419, "count": 222, "max_ms": 138.308, "p95_ms": 110.329}, "visualize": {"avg_ms": 5.102, "count": 222, "max_ms": 6.925, "p95_ms": 6.018}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4742.293, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:16:40+09:00", "last_error_summary": "[NULL @ 0x55d650d0e200] missing picture in access unit with size 18 [h264 @ 0x55d650d8ab40] no frame! [vist#0:0/h264 @ 0x55d650d0e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T12:16:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 729.514, "last_started_at": "2026-07-06T12:04:31+09:00", "pid": 4334, "process": {"exists": false, "pid": 4334}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.696, "sample_count": 2233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28457, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T12:16:40+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T12:16:40+09:00", "mjpeg": {"age_sec": 0.145, "exists": true, "mtime": "2026-07-06T12:16:40.831894+09:00", "mtime_epoch": 1783307800.8318937, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236823}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:16:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1071.02, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.439, "pushed": 7.439, "read": 7.406}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.296, "count": 222, "max_ms": 9.01, "p95_ms": 7.953}, "gstreamer_push": {"avg_ms": 1.126, "count": 223, "max_ms": 3.702, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.465, "count": 222, "max_ms": 82.878, "p95_ms": 66.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.374, "count": 222, "max_ms": 138.296, "p95_ms": 106.393}, "yolo11_640_fire_11class": {"avg_ms": 60.395, "count": 222, "max_ms": 91.47, "p95_ms": 78.831}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.752, "count": 222, "max_ms": 122.887, "p95_ms": 107.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.156, "count": 223, "max_ms": 193.539, "p95_ms": 163.336}, "read_interval": {"avg_ms": 134.903, "count": 222, "max_ms": 204.237, "p95_ms": 174.759}, "triton_wait": {"avg_ms": 72.419, "count": 222, "max_ms": 138.308, "p95_ms": 110.329}, "visualize": {"avg_ms": 5.102, "count": 222, "max_ms": 6.925, "p95_ms": 6.018}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4919.113, "limit_mb": 8796093022207.996}, "detail": {"pid": 5261, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:16:40+09:00", "last_error_summary": "[NULL @ 0x55d650d0e200] missing picture in access unit with size 18 [h264 @ 0x55d650d8ab40] no frame! [vist#0:0/h264 @ 0x55d650d0e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T12:16:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:16:43+09:00", "pid": 5261, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.527, "name": "ffmpeg", "pid": 5261, "rss_mb": 197.527, "state": "S (sleeping)", "threads": 106}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.696, "sample_count": 2197, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28457, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 4.93, "last_frame_pushed_at": "2026-07-06T12:16:40+09:00", "last_frame_read_age_sec": 5.064, "last_frame_read_at": "2026-07-06T12:16:40+09:00", "mjpeg": {"age_sec": 4.905, "exists": true, "mtime": "2026-07-06T12:16:40.991895+09:00", "mtime_epoch": 1783307800.9918954, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 235353}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:16:45+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1071.02, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 187, "pushed_frames": 187, "read_frames": 186, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.238, "pushed": 6.238, "read": 6.204}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.304, "count": 186, "max_ms": 9.01, "p95_ms": 7.79}, "gstreamer_push": {"avg_ms": 1.133, "count": 187, "max_ms": 3.702, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.009, "count": 186, "max_ms": 78.891, "p95_ms": 65.323}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.963, "count": 186, "max_ms": 117.76, "p95_ms": 105.449}, "yolo11_640_fire_11class": {"avg_ms": 60.293, "count": 186, "max_ms": 91.47, "p95_ms": 78.831}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.432, "count": 186, "max_ms": 122.887, "p95_ms": 105.952}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.83, "count": 187, "max_ms": 177.243, "p95_ms": 163.336}, "read_interval": {"avg_ms": 134.546, "count": 186, "max_ms": 189.243, "p95_ms": 172.484}, "triton_wait": {"avg_ms": 72.088, "count": 186, "max_ms": 122.899, "p95_ms": 110.329}, "visualize": {"avg_ms": 5.106, "count": 186, "max_ms": 6.925, "p95_ms": 6.018}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 48, "tracking_summary": 132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4812.035, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ee7eb14700] error while decoding MB 6 38, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:17:52+09:00", "last_error_summary": "[h264 @ 0x55ee7eb14700] error while decoding MB 6 38, bytestream -5", "last_exit_at": "2026-07-06T12:15:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.964, "last_started_at": "2026-07-06T12:15:50+09:00", "pid": 5154, "process": {"exists": false, "pid": 5154}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 743.059, "sample_count": 667, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15865, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:17:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.042, "last_frame_pushed_at": "2026-07-06T12:17:52+09:00", "last_frame_read_age_sec": 0.038, "last_frame_read_at": "2026-07-06T12:17:52+09:00", "mjpeg": {"age_sec": 0.025, "exists": true, "mtime": "2026-07-06T12:17:52.224734+09:00", "mtime_epoch": 1783307872.2247338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196279}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:17:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1163.801, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.187, "pushed": 3.187, "read": 3.187}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 17.481, "count": 94, "max_ms": 20.593, "p95_ms": 20.144}, "gstreamer_push": {"avg_ms": 0.882, "count": 95, "max_ms": 1.499, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.695, "count": 94, "max_ms": 82.722, "p95_ms": 73.148}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.749, "count": 94, "max_ms": 125.513, "p95_ms": 111.311}, "yolo11_640_fire_11class": {"avg_ms": 57.019, "count": 94, "max_ms": 88.596, "p95_ms": 79.039}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.807, "count": 94, "max_ms": 122.561, "p95_ms": 113.102}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.379, "count": 95, "max_ms": 374.795, "p95_ms": 359.19}, "read_interval": {"avg_ms": 316.882, "count": 95, "max_ms": 382.833, "p95_ms": 366.468}, "triton_wait": {"avg_ms": 81.45, "count": 94, "max_ms": 125.526, "p95_ms": 114.884}, "visualize": {"avg_ms": 13.416, "count": 94, "max_ms": 18.787, "p95_ms": 15.462}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.811, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 48, "tracking_summary": 132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4811.203, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:17:52+09:00", "last_error_summary": "[h264 @ 0x55ee7eb14700] error while decoding MB 6 38, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:17:52+09:00", "last_error_summary": "[h264 @ 0x55ee7eb14700] error while decoding MB 6 38, bytestream -5", "last_exit_at": "2026-07-06T12:17:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.209, "last_started_at": "2026-07-06T12:15:50+09:00", "pid": 5154, "process": {"exists": false, "pid": 5154}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 743.059, "sample_count": 668, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15866, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:17:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T12:17:52+09:00", "last_frame_read_age_sec": 0.283, "last_frame_read_at": "2026-07-06T12:17:52+09:00", "mjpeg": {"age_sec": 0.27, "exists": true, "mtime": "2026-07-06T12:17:52.224734+09:00", "mtime_epoch": 1783307872.2247338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196279}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:17:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1163.801, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.194, "pushed": 3.194, "read": 3.16}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 17.459, "count": 95, "max_ms": 20.593, "p95_ms": 20.144}, "gstreamer_push": {"avg_ms": 0.881, "count": 95, "max_ms": 1.499, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.717, "count": 95, "max_ms": 82.722, "p95_ms": 73.148}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.661, "count": 95, "max_ms": 125.513, "p95_ms": 111.311}, "yolo11_640_fire_11class": {"avg_ms": 56.997, "count": 95, "max_ms": 88.596, "p95_ms": 79.039}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.728, "count": 95, "max_ms": 122.561, "p95_ms": 113.102}}, "truncated_count": 0}, "processing_total": {"avg_ms": 309.819, "count": 95, "max_ms": 374.795, "p95_ms": 359.19}, "read_interval": {"avg_ms": 316.674, "count": 94, "max_ms": 382.833, "p95_ms": 366.468}, "triton_wait": {"avg_ms": 81.355, "count": 95, "max_ms": 125.526, "p95_ms": 114.884}, "visualize": {"avg_ms": 13.407, "count": 95, "max_ms": 18.787, "p95_ms": 15.462}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.747, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 48, "tracking_summary": 132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4811.195, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:17:52+09:00", "last_error_summary": "[h264 @ 0x55ee7eb14700] error while decoding MB 6 38, bytestream -5", "last_exit_at": "2026-07-06T12:17:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.212, "last_started_at": "2026-07-06T12:15:50+09:00", "pid": 5154, "process": {"exists": false, "pid": 5154}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 743.059, "sample_count": 668, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15866, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:17:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T12:17:52+09:00", "last_frame_read_age_sec": 0.286, "last_frame_read_at": "2026-07-06T12:17:52+09:00", "mjpeg": {"age_sec": 0.273, "exists": true, "mtime": "2026-07-06T12:17:52.224734+09:00", "mtime_epoch": 1783307872.2247338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196279}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:17:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1163.801, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.193, "pushed": 3.193, "read": 3.16}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 17.459, "count": 95, "max_ms": 20.593, "p95_ms": 20.144}, "gstreamer_push": {"avg_ms": 0.881, "count": 95, "max_ms": 1.499, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.717, "count": 95, "max_ms": 82.722, "p95_ms": 73.148}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.661, "count": 95, "max_ms": 125.513, "p95_ms": 111.311}, "yolo11_640_fire_11class": {"avg_ms": 56.997, "count": 95, "max_ms": 88.596, "p95_ms": 79.039}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.728, "count": 95, "max_ms": 122.561, "p95_ms": 113.102}}, "truncated_count": 0}, "processing_total": {"avg_ms": 309.819, "count": 95, "max_ms": 374.795, "p95_ms": 359.19}, "read_interval": {"avg_ms": 316.674, "count": 94, "max_ms": 382.833, "p95_ms": 366.468}, "triton_wait": {"avg_ms": 81.355, "count": 95, "max_ms": 125.526, "p95_ms": 114.884}, "visualize": {"avg_ms": 13.407, "count": 95, "max_ms": 18.787, "p95_ms": 15.462}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.749, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 48, "tracking_summary": 132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4932.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 5413, "restart_count": 22, "start_count": 23}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:17:52+09:00", "last_error_summary": "[h264 @ 0x55ee7eb14700] error while decoding MB 6 38, bytestream -5", "last_exit_at": "2026-07-06T12:17:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:17:54+09:00", "pid": 5413, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.684, "name": "ffmpeg", "pid": 5413, "rss_mb": 141.684, "state": "S (sleeping)", "threads": 106}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 725.443, "sample_count": 662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15866, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:17:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.16, "last_frame_pushed_at": "2026-07-06T12:17:52+09:00", "last_frame_read_age_sec": 4.439, "last_frame_read_at": "2026-07-06T12:17:52+09:00", "mjpeg": {"age_sec": 4.142, "exists": true, "mtime": "2026-07-06T12:17:52.508737+09:00", "mtime_epoch": 1783307872.5087373, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196441}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:17:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1163.801, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.774, "pushed": 2.774, "read": 2.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 17.223, "count": 82, "max_ms": 20.244, "p95_ms": 19.281}, "gstreamer_push": {"avg_ms": 0.87, "count": 83, "max_ms": 1.333, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.799, "count": 82, "max_ms": 82.722, "p95_ms": 71.92}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.52, "count": 82, "max_ms": 125.513, "p95_ms": 111.115}, "yolo11_640_fire_11class": {"avg_ms": 57.342, "count": 82, "max_ms": 88.596, "p95_ms": 78.442}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.842, "count": 82, "max_ms": 122.561, "p95_ms": 111.127}}, "truncated_count": 0}, "processing_total": {"avg_ms": 307.732, "count": 83, "max_ms": 374.795, "p95_ms": 357.451}, "read_interval": {"avg_ms": 314.589, "count": 82, "max_ms": 382.833, "p95_ms": 365.111}, "triton_wait": {"avg_ms": 81.28, "count": 82, "max_ms": 125.526, "p95_ms": 113.115}, "visualize": {"avg_ms": 13.261, "count": 82, "max_ms": 18.787, "p95_ms": 15.085}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4756.156, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556e0cdd4d40] error while decoding MB 91 35, bytestream -5 [in#0/rtsp @ 0x556e0c263e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556e0c263e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:18:13+09:00", "last_error_summary": "[h264 @ 0x556e0cdd4d40] error while decoding MB 91 35, bytestream -5 [in#0/rtsp @ 0x556e0c263e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556e0c263e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T12:16:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 89.74, "last_started_at": "2026-07-06T12:16:43+09:00", "pid": 5261, "process": {"exists": false, "pid": 5261}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.472, "sample_count": 1938, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:18:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T12:18:13+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-06T12:18:13+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T12:18:13.584994+09:00", "mtime_epoch": 1783307893.5849936, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 214523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:18:13+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1073.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.654, "pushed": 0.654, "read": 0.709}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.961, "count": 12, "max_ms": 8.683, "p95_ms": 8.683}, "gstreamer_push": {"avg_ms": 1.216, "count": 12, "max_ms": 2.796, "p95_ms": 2.796}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.252, "count": 12, "max_ms": 64.654, "p95_ms": 64.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.417, "count": 12, "max_ms": 116.594, "p95_ms": 116.594}, "yolo11_640_fire_11class": {"avg_ms": 61.907, "count": 12, "max_ms": 87.588, "p95_ms": 87.588}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.406, "count": 12, "max_ms": 118.809, "p95_ms": 118.809}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.911, "count": 12, "max_ms": 149.945, "p95_ms": 149.945}, "read_interval": {"avg_ms": 2704.181, "count": 13, "max_ms": 16874.921, "p95_ms": 16874.921}, "triton_wait": {"avg_ms": 75.744, "count": 12, "max_ms": 118.821, "p95_ms": 118.821}, "visualize": {"avg_ms": 4.404, "count": 12, "max_ms": 5.568, "p95_ms": 5.568}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.339, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4755.059, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:18:13+09:00", "last_error_summary": "[h264 @ 0x556e0cdd4d40] error while decoding MB 91 35, bytestream -5 [in#0/rtsp @ 0x556e0c263e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556e0c263e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:18:13+09:00", "last_error_summary": "[h264 @ 0x556e0cdd4d40] error while decoding MB 91 35, bytestream -5 [in#0/rtsp @ 0x556e0c263e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556e0c263e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T12:18:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 89.79, "last_started_at": "2026-07-06T12:16:43+09:00", "pid": 5261, "process": {"exists": false, "pid": 5261}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.472, "sample_count": 1939, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:18:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T12:18:13+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T12:18:13+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T12:18:13.584994+09:00", "mtime_epoch": 1783307893.5849936, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 214523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:18:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1073.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.707, "pushed": 0.707, "read": 0.707}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.793, "count": 13, "max_ms": 8.683, "p95_ms": 8.683}, "gstreamer_push": {"avg_ms": 1.222, "count": 13, "max_ms": 2.796, "p95_ms": 2.796}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.208, "count": 13, "max_ms": 64.654, "p95_ms": 64.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.434, "count": 13, "max_ms": 116.594, "p95_ms": 116.594}, "yolo11_640_fire_11class": {"avg_ms": 62.307, "count": 13, "max_ms": 87.588, "p95_ms": 87.588}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.418, "count": 13, "max_ms": 118.809, "p95_ms": 118.809}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.39, "count": 13, "max_ms": 149.945, "p95_ms": 149.945}, "read_interval": {"avg_ms": 2704.181, "count": 13, "max_ms": 16874.921, "p95_ms": 16874.921}, "triton_wait": {"avg_ms": 76.737, "count": 13, "max_ms": 118.821, "p95_ms": 118.821}, "visualize": {"avg_ms": 4.338, "count": 13, "max_ms": 5.568, "p95_ms": 5.568}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.388, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4755.02, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:18:13+09:00", "last_error_summary": "[h264 @ 0x556e0cdd4d40] error while decoding MB 91 35, bytestream -5 [in#0/rtsp @ 0x556e0c263e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556e0c263e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T12:18:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 89.796, "last_started_at": "2026-07-06T12:16:43+09:00", "pid": 5261, "process": {"exists": false, "pid": 5261}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.472, "sample_count": 1939, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:18:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T12:18:13+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T12:18:13+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T12:18:13.584994+09:00", "mtime_epoch": 1783307893.5849936, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 214523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:18:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1073.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.707, "pushed": 0.707, "read": 0.707}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.793, "count": 13, "max_ms": 8.683, "p95_ms": 8.683}, "gstreamer_push": {"avg_ms": 1.222, "count": 13, "max_ms": 2.796, "p95_ms": 2.796}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.208, "count": 13, "max_ms": 64.654, "p95_ms": 64.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.434, "count": 13, "max_ms": 116.594, "p95_ms": 116.594}, "yolo11_640_fire_11class": {"avg_ms": 62.307, "count": 13, "max_ms": 87.588, "p95_ms": 87.588}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.418, "count": 13, "max_ms": 118.809, "p95_ms": 118.809}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.39, "count": 13, "max_ms": 149.945, "p95_ms": 149.945}, "read_interval": {"avg_ms": 2704.181, "count": 13, "max_ms": 16874.921, "p95_ms": 16874.921}, "triton_wait": {"avg_ms": 76.737, "count": 13, "max_ms": 118.821, "p95_ms": 118.821}, "visualize": {"avg_ms": 4.338, "count": 13, "max_ms": 5.568, "p95_ms": 5.568}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.395, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4834.344, "limit_mb": 8796093022207.996}, "detail": {"pid": 5520, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:18:13+09:00", "last_error_summary": "[h264 @ 0x556e0cdd4d40] error while decoding MB 91 35, bytestream -5 [in#0/rtsp @ 0x556e0c263e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556e0c263e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T12:18:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:18:16+09:00", "pid": 5520, "process": {"exists": true, "fd_count": 5, "hwm_mb": 109.352, "name": "ffmpeg", "pid": 5520, "rss_mb": 109.352, "state": "S (sleeping)", "threads": 19}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.585, "sample_count": 1901, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:18:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 4.921, "last_frame_pushed_at": "2026-07-06T12:18:13+09:00", "last_frame_read_age_sec": 5.024, "last_frame_read_at": "2026-07-06T12:18:13+09:00", "mjpeg": {"age_sec": 4.903, "exists": true, "mtime": "2026-07-06T12:18:13.696995+09:00", "mtime_epoch": 1783307893.6969948, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 212884}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:18:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1073.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.558, "pushed": 0.558, "read": 0.558}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.793, "count": 13, "max_ms": 8.683, "p95_ms": 8.683}, "gstreamer_push": {"avg_ms": 1.222, "count": 13, "max_ms": 2.796, "p95_ms": 2.796}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.208, "count": 13, "max_ms": 64.654, "p95_ms": 64.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.434, "count": 13, "max_ms": 116.594, "p95_ms": 116.594}, "yolo11_640_fire_11class": {"avg_ms": 62.307, "count": 13, "max_ms": 87.588, "p95_ms": 87.588}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.418, "count": 13, "max_ms": 118.809, "p95_ms": 118.809}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.39, "count": 13, "max_ms": 149.945, "p95_ms": 149.945}, "read_interval": {"avg_ms": 2704.181, "count": 13, "max_ms": 16874.921, "p95_ms": 16874.921}, "triton_wait": {"avg_ms": 76.737, "count": 13, "max_ms": 118.821, "p95_ms": 118.821}, "visualize": {"avg_ms": 4.338, "count": 13, "max_ms": 5.568, "p95_ms": 5.568}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.303, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4827.426, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:19:46+09:00", "last_error_summary": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "last_exit_at": "2026-07-06T12:17:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.844, "last_started_at": "2026-07-06T12:17:54+09:00", "pid": 5413, "process": {"exists": false, "pid": 5413}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 508.869, "sample_count": 878, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16262, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:19:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T12:19:46+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T12:19:46+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-06T12:19:46.458162+09:00", "mtime_epoch": 1783307986.4581616, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:19:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.059, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.847, "pushed": 3.847, "read": 3.847}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.567, "count": 114, "max_ms": 23.125, "p95_ms": 19.917}, "gstreamer_push": {"avg_ms": 0.877, "count": 115, "max_ms": 1.087, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.946, "count": 114, "max_ms": 79.979, "p95_ms": 75.512}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.189, "count": 114, "max_ms": 117.46, "p95_ms": 112.288}, "yolo11_640_fire_11class": {"avg_ms": 61.495, "count": 114, "max_ms": 88.873, "p95_ms": 85.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.551, "count": 114, "max_ms": 116.626, "p95_ms": 107.156}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.095, "count": 115, "max_ms": 345.004, "p95_ms": 325.353}, "read_interval": {"avg_ms": 261.882, "count": 115, "max_ms": 351.508, "p95_ms": 331.776}, "triton_wait": {"avg_ms": 83.092, "count": 114, "max_ms": 117.476, "p95_ms": 114.845}, "visualize": {"avg_ms": 11.931, "count": 114, "max_ms": 19.247, "p95_ms": 17.627}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4828.93, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:19:46+09:00", "last_error_summary": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:19:46+09:00", "last_error_summary": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "last_exit_at": "2026-07-06T12:19:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.988, "last_started_at": "2026-07-06T12:17:54+09:00", "pid": 5413, "process": {"exists": false, "pid": 5413}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 508.462, "sample_count": 878, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16263, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:19:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T12:19:46+09:00", "last_frame_read_age_sec": 0.196, "last_frame_read_at": "2026-07-06T12:19:46+09:00", "mjpeg": {"age_sec": 0.18, "exists": true, "mtime": "2026-07-06T12:19:46.458162+09:00", "mtime_epoch": 1783307986.4581616, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:19:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.059, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.871, "pushed": 3.871, "read": 3.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.537, "count": 115, "max_ms": 23.125, "p95_ms": 19.917}, "gstreamer_push": {"avg_ms": 0.876, "count": 115, "max_ms": 1.087, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.879, "count": 115, "max_ms": 79.979, "p95_ms": 75.512}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.048, "count": 115, "max_ms": 117.46, "p95_ms": 112.288}, "yolo11_640_fire_11class": {"avg_ms": 61.408, "count": 115, "max_ms": 88.873, "p95_ms": 85.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.39, "count": 115, "max_ms": 116.626, "p95_ms": 107.156}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.403, "count": 115, "max_ms": 345.004, "p95_ms": 325.353}, "read_interval": {"avg_ms": 261.739, "count": 114, "max_ms": 351.508, "p95_ms": 331.776}, "triton_wait": {"avg_ms": 82.943, "count": 115, "max_ms": 117.476, "p95_ms": 114.845}, "visualize": {"avg_ms": 11.898, "count": 115, "max_ms": 19.247, "p95_ms": 17.627}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.707, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4828.812, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:19:46+09:00", "last_error_summary": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "last_exit_at": "2026-07-06T12:19:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.991, "last_started_at": "2026-07-06T12:17:54+09:00", "pid": 5413, "process": {"exists": false, "pid": 5413}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 508.462, "sample_count": 878, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16263, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:19:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T12:19:46+09:00", "last_frame_read_age_sec": 0.2, "last_frame_read_at": "2026-07-06T12:19:46+09:00", "mjpeg": {"age_sec": 0.183, "exists": true, "mtime": "2026-07-06T12:19:46.458162+09:00", "mtime_epoch": 1783307986.4581616, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:19:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.059, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.871, "pushed": 3.871, "read": 3.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.537, "count": 115, "max_ms": 23.125, "p95_ms": 19.917}, "gstreamer_push": {"avg_ms": 0.876, "count": 115, "max_ms": 1.087, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.879, "count": 115, "max_ms": 79.979, "p95_ms": 75.512}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.048, "count": 115, "max_ms": 117.46, "p95_ms": 112.288}, "yolo11_640_fire_11class": {"avg_ms": 61.408, "count": 115, "max_ms": 88.873, "p95_ms": 85.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.39, "count": 115, "max_ms": 116.626, "p95_ms": 107.156}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.403, "count": 115, "max_ms": 345.004, "p95_ms": 325.353}, "read_interval": {"avg_ms": 261.739, "count": 114, "max_ms": 351.508, "p95_ms": 331.776}, "triton_wait": {"avg_ms": 82.943, "count": 115, "max_ms": 117.476, "p95_ms": 114.845}, "visualize": {"avg_ms": 11.898, "count": 115, "max_ms": 19.247, "p95_ms": 17.627}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.71, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4925.844, "limit_mb": 8796093022207.996}, "detail": {"pid": 5732, "restart_count": 23, "start_count": 24}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:19:46+09:00", "last_error_summary": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "last_exit_at": "2026-07-06T12:19:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:19:48+09:00", "pid": 5732, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.379, "name": "ffmpeg", "pid": 5732, "rss_mb": 139.379, "state": "S (sleeping)", "threads": 106}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 506.53, "sample_count": 870, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16263, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:19:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.059, "last_frame_pushed_at": "2026-07-06T12:19:46+09:00", "last_frame_read_age_sec": 4.253, "last_frame_read_at": "2026-07-06T12:19:46+09:00", "mjpeg": {"age_sec": 4.041, "exists": true, "mtime": "2026-07-06T12:19:46.654164+09:00", "mtime_epoch": 1783307986.654164, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191677}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:19:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.059, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.407, "pushed": 3.407, "read": 3.374}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.186, "count": 101, "max_ms": 21.553, "p95_ms": 18.731}, "gstreamer_push": {"avg_ms": 0.878, "count": 102, "max_ms": 1.087, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.444, "count": 101, "max_ms": 79.979, "p95_ms": 74.609}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.713, "count": 101, "max_ms": 117.46, "p95_ms": 111.623}, "yolo11_640_fire_11class": {"avg_ms": 61.267, "count": 101, "max_ms": 88.873, "p95_ms": 84.168}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.13, "count": 101, "max_ms": 114.832, "p95_ms": 103.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.173, "count": 102, "max_ms": 345.004, "p95_ms": 325.118}, "read_interval": {"avg_ms": 257.526, "count": 101, "max_ms": 351.508, "p95_ms": 331.6}, "triton_wait": {"avg_ms": 82.669, "count": 101, "max_ms": 117.476, "p95_ms": 111.716}, "visualize": {"avg_ms": 11.305, "count": 101, "max_ms": 17.774, "p95_ms": 17.318}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4834.668, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:19:46+09:00", "last_error_summary": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:19:46+09:00", "last_error_summary": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "last_exit_at": "2026-07-06T12:21:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.093, "last_started_at": "2026-07-06T12:19:48+09:00", "pid": 5732, "process": {"exists": false, "pid": 5732}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 345.004, "sample_count": 1215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16901, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:21:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T12:21:52+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-06T12:21:52+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-06T12:21:52.583835+09:00", "mtime_epoch": 1783308112.5838351, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201792}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:21:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.574, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 157, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.285, "pushed": 5.285, "read": 5.252}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.549, "count": 158, "max_ms": 14.227, "p95_ms": 13.045}, "gstreamer_push": {"avg_ms": 0.917, "count": 158, "max_ms": 1.235, "p95_ms": 1.06}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.155, "count": 158, "max_ms": 81.334, "p95_ms": 69.911}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.295, "count": 158, "max_ms": 116.967, "p95_ms": 103.31}, "yolo11_640_fire_11class": {"avg_ms": 57.285, "count": 158, "max_ms": 93.655, "p95_ms": 80.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.523, "count": 158, "max_ms": 109.6, "p95_ms": 102.095}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.225, "count": 158, "max_ms": 244.708, "p95_ms": 220.123}, "read_interval": {"avg_ms": 190.22, "count": 157, "max_ms": 250.965, "p95_ms": 227.426}, "triton_wait": {"avg_ms": 75.226, "count": 158, "max_ms": 116.98, "p95_ms": 104.113}, "visualize": {"avg_ms": 7.688, "count": 158, "max_ms": 10.397, "p95_ms": 9.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4834.66, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:19:46+09:00", "last_error_summary": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "last_exit_at": "2026-07-06T12:21:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.097, "last_started_at": "2026-07-06T12:19:48+09:00", "pid": 5732, "process": {"exists": false, "pid": 5732}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 345.004, "sample_count": 1215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16901, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:21:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T12:21:52+09:00", "last_frame_read_age_sec": 0.218, "last_frame_read_at": "2026-07-06T12:21:52+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-06T12:21:52.583835+09:00", "mtime_epoch": 1783308112.5838351, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201792}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:21:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.574, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 157, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.285, "pushed": 5.285, "read": 5.251}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.546, "count": 157, "max_ms": 14.227, "p95_ms": 13.045}, "gstreamer_push": {"avg_ms": 0.917, "count": 158, "max_ms": 1.235, "p95_ms": 1.06}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.225, "count": 157, "max_ms": 81.334, "p95_ms": 69.911}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.402, "count": 157, "max_ms": 116.967, "p95_ms": 103.31}, "yolo11_640_fire_11class": {"avg_ms": 57.346, "count": 157, "max_ms": 93.655, "p95_ms": 80.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.589, "count": 157, "max_ms": 109.6, "p95_ms": 102.095}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.225, "count": 158, "max_ms": 244.708, "p95_ms": 220.123}, "read_interval": {"avg_ms": 190.22, "count": 157, "max_ms": 250.965, "p95_ms": 227.426}, "triton_wait": {"avg_ms": 75.328, "count": 157, "max_ms": 116.98, "p95_ms": 104.113}, "visualize": {"avg_ms": 7.685, "count": 157, "max_ms": 10.397, "p95_ms": 9.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4937.738, "limit_mb": 8796093022207.996}, "detail": {"pid": 5839, "restart_count": 24, "start_count": 25}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:19:46+09:00", "last_error_summary": "[h264 @ 0x55f2da6f9240] error while decoding MB 63 18, bytestream -11", "last_exit_at": "2026-07-06T12:21:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:21:54+09:00", "pid": 5839, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.383, "name": "ffmpeg", "pid": 5839, "rss_mb": 142.383, "state": "S (sleeping)", "threads": 106}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 342.18, "sample_count": 1204, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16901, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:21:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.053, "last_frame_pushed_at": "2026-07-06T12:21:52+09:00", "last_frame_read_age_sec": 4.263, "last_frame_read_at": "2026-07-06T12:21:52+09:00", "mjpeg": {"age_sec": 4.037, "exists": true, "mtime": "2026-07-06T12:21:52.799838+09:00", "mtime_epoch": 1783308112.799838, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:21:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.574, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.582, "pushed": 4.582, "read": 4.548}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.526, "count": 137, "max_ms": 14.227, "p95_ms": 13.388}, "gstreamer_push": {"avg_ms": 0.919, "count": 137, "max_ms": 1.235, "p95_ms": 1.075}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.714, "count": 137, "max_ms": 81.334, "p95_ms": 68.352}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.41, "count": 137, "max_ms": 116.967, "p95_ms": 104.101}, "yolo11_640_fire_11class": {"avg_ms": 57.01, "count": 137, "max_ms": 92.148, "p95_ms": 80.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.085, "count": 137, "max_ms": 109.6, "p95_ms": 102.095}}, "truncated_count": 0}, "processing_total": {"avg_ms": 182.719, "count": 137, "max_ms": 244.708, "p95_ms": 218.735}, "read_interval": {"avg_ms": 189.707, "count": 136, "max_ms": 250.965, "p95_ms": 226.875}, "triton_wait": {"avg_ms": 75.232, "count": 137, "max_ms": 116.98, "p95_ms": 104.94}, "visualize": {"avg_ms": 7.677, "count": 137, "max_ms": 10.397, "p95_ms": 9.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4835.898, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5627c269d840] error while decoding MB 34 38, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:24:02+09:00", "last_error_summary": "[h264 @ 0x5627c269d840] error while decoding MB 34 38, bytestream -5", "last_exit_at": "2026-07-06T12:21:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.681, "last_started_at": "2026-07-06T12:21:54+09:00", "pid": 5839, "process": {"exists": false, "pid": 5839}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.437, "sample_count": 1516, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17619, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:24:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T12:24:02+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T12:24:02+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T12:24:02.473643+09:00", "mtime_epoch": 1783308242.4736426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192397}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:24:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.832, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 195, "pushed_frames": 195, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.528, "pushed": 6.528, "read": 6.528}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.66, "count": 195, "max_ms": 13.162, "p95_ms": 10.719}, "gstreamer_push": {"avg_ms": 1.039, "count": 195, "max_ms": 2.012, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.716, "count": 195, "max_ms": 86.737, "p95_ms": 77.208}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.852, "count": 195, "max_ms": 124.565, "p95_ms": 107.715}, "yolo11_640_fire_11class": {"avg_ms": 59.237, "count": 195, "max_ms": 86.229, "p95_ms": 81.172}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.321, "count": 195, "max_ms": 111.016, "p95_ms": 104.545}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.019, "count": 195, "max_ms": 223.215, "p95_ms": 184.64}, "read_interval": {"avg_ms": 153.779, "count": 195, "max_ms": 230.134, "p95_ms": 192.186}, "triton_wait": {"avg_ms": 79.423, "count": 195, "max_ms": 124.58, "p95_ms": 109.318}, "visualize": {"avg_ms": 5.638, "count": 195, "max_ms": 9.254, "p95_ms": 7.385}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4834.895, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:24:02+09:00", "last_error_summary": "[h264 @ 0x5627c269d840] error while decoding MB 34 38, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:24:02+09:00", "last_error_summary": "[h264 @ 0x5627c269d840] error while decoding MB 34 38, bytestream -5", "last_exit_at": "2026-07-06T12:24:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.799, "last_started_at": "2026-07-06T12:21:54+09:00", "pid": 5839, "process": {"exists": false, "pid": 5839}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.437, "sample_count": 1517, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17620, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:24:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T12:24:02+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-06T12:24:02+09:00", "mjpeg": {"age_sec": 0.16, "exists": true, "mtime": "2026-07-06T12:24:02.473643+09:00", "mtime_epoch": 1783308242.4736426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192397}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:24:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.832, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.536, "pushed": 6.536, "read": 6.503}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.657, "count": 195, "max_ms": 13.162, "p95_ms": 10.719}, "gstreamer_push": {"avg_ms": 1.04, "count": 196, "max_ms": 2.012, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.781, "count": 195, "max_ms": 86.737, "p95_ms": 77.208}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.965, "count": 195, "max_ms": 124.565, "p95_ms": 107.715}, "yolo11_640_fire_11class": {"avg_ms": 59.216, "count": 195, "max_ms": 86.229, "p95_ms": 81.172}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.342, "count": 195, "max_ms": 111.016, "p95_ms": 104.545}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.104, "count": 196, "max_ms": 223.215, "p95_ms": 184.64}, "read_interval": {"avg_ms": 153.779, "count": 195, "max_ms": 230.134, "p95_ms": 192.186}, "triton_wait": {"avg_ms": 79.536, "count": 195, "max_ms": 124.58, "p95_ms": 109.318}, "visualize": {"avg_ms": 5.631, "count": 195, "max_ms": 9.254, "p95_ms": 7.385}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4834.887, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:24:02+09:00", "last_error_summary": "[h264 @ 0x5627c269d840] error while decoding MB 34 38, bytestream -5", "last_exit_at": "2026-07-06T12:24:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.804, "last_started_at": "2026-07-06T12:21:54+09:00", "pid": 5839, "process": {"exists": false, "pid": 5839}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.437, "sample_count": 1517, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17620, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:24:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T12:24:02+09:00", "last_frame_read_age_sec": 0.174, "last_frame_read_at": "2026-07-06T12:24:02+09:00", "mjpeg": {"age_sec": 0.166, "exists": true, "mtime": "2026-07-06T12:24:02.473643+09:00", "mtime_epoch": 1783308242.4736426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192397}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:24:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.832, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.535, "pushed": 6.535, "read": 6.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.657, "count": 195, "max_ms": 13.162, "p95_ms": 10.719}, "gstreamer_push": {"avg_ms": 1.04, "count": 196, "max_ms": 2.012, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.781, "count": 195, "max_ms": 86.737, "p95_ms": 77.208}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.965, "count": 195, "max_ms": 124.565, "p95_ms": 107.715}, "yolo11_640_fire_11class": {"avg_ms": 59.216, "count": 195, "max_ms": 86.229, "p95_ms": 81.172}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.342, "count": 195, "max_ms": 111.016, "p95_ms": 104.545}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.104, "count": 196, "max_ms": 223.215, "p95_ms": 184.64}, "read_interval": {"avg_ms": 153.779, "count": 195, "max_ms": 230.134, "p95_ms": 192.186}, "triton_wait": {"avg_ms": 79.536, "count": 195, "max_ms": 124.58, "p95_ms": 109.318}, "visualize": {"avg_ms": 5.631, "count": 195, "max_ms": 9.254, "p95_ms": 7.385}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4950.285, "limit_mb": 8796093022207.996}, "detail": {"pid": 5946, "restart_count": 25, "start_count": 26}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:24:02+09:00", "last_error_summary": "[h264 @ 0x5627c269d840] error while decoding MB 34 38, bytestream -5", "last_exit_at": "2026-07-06T12:24:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:24:04+09:00", "pid": 5946, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.484, "name": "ffmpeg", "pid": 5946, "rss_mb": 147.484, "state": "S (sleeping)", "threads": 106}, "restart_count": 25, "restart_count_per_hour": 25, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.464, "sample_count": 1505, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17620, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:24:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.169, "last_frame_pushed_at": "2026-07-06T12:24:02+09:00", "last_frame_read_age_sec": 4.332, "last_frame_read_at": "2026-07-06T12:24:02+09:00", "mjpeg": {"age_sec": 4.152, "exists": true, "mtime": "2026-07-06T12:24:02.645645+09:00", "mtime_epoch": 1783308242.645645, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192807}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:24:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1164.832, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 168, "pushed_frames": 168, "read_frames": 167, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.625, "pushed": 5.625, "read": 5.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.721, "count": 168, "max_ms": 13.162, "p95_ms": 10.719}, "gstreamer_push": {"avg_ms": 1.046, "count": 168, "max_ms": 2.012, "p95_ms": 1.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.02, "count": 168, "max_ms": 86.737, "p95_ms": 77.529}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.13, "count": 168, "max_ms": 124.565, "p95_ms": 107.715}, "yolo11_640_fire_11class": {"avg_ms": 59.331, "count": 168, "max_ms": 86.229, "p95_ms": 80.968}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.847, "count": 168, "max_ms": 111.016, "p95_ms": 104.855}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.171, "count": 168, "max_ms": 223.215, "p95_ms": 181.385}, "read_interval": {"avg_ms": 153.814, "count": 167, "max_ms": 230.134, "p95_ms": 189.976}, "triton_wait": {"avg_ms": 79.79, "count": 168, "max_ms": 124.58, "p95_ms": 109.318}, "visualize": {"avg_ms": 5.668, "count": 168, "max_ms": 9.254, "p95_ms": 7.405}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4838.957, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b73a5e5240] error while decoding MB 40 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:26:05+09:00", "last_error_summary": "[h264 @ 0x55b73a5e5240] error while decoding MB 40 37, bytestream -5", "last_exit_at": "2026-07-06T12:24:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.97, "last_started_at": "2026-07-06T12:24:04+09:00", "pid": 5946, "process": {"exists": false, "pid": 5946}, "restart_count": 25, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.03, "sample_count": 1540, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18198, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:26:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T12:26:05+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T12:26:05+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T12:26:05.723419+09:00", "mtime_epoch": 1783308365.723419, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:26:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1165.09, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.367, "pushed": 5.367, "read": 5.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.478, "count": 160, "max_ms": 16.406, "p95_ms": 12.426}, "gstreamer_push": {"avg_ms": 0.932, "count": 160, "max_ms": 1.263, "p95_ms": 1.077}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.58, "count": 160, "max_ms": 87.973, "p95_ms": 76.182}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.355, "count": 160, "max_ms": 132.947, "p95_ms": 107.444}, "yolo11_640_fire_11class": {"avg_ms": 57.307, "count": 160, "max_ms": 88.649, "p95_ms": 83.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.836, "count": 160, "max_ms": 116.181, "p95_ms": 103.506}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.815, "count": 160, "max_ms": 244.921, "p95_ms": 224.506}, "read_interval": {"avg_ms": 187.251, "count": 160, "max_ms": 254.233, "p95_ms": 233.196}, "triton_wait": {"avg_ms": 76.617, "count": 160, "max_ms": 132.961, "p95_ms": 107.585}, "visualize": {"avg_ms": 7.489, "count": 160, "max_ms": 9.613, "p95_ms": 9.066}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.814, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4829.566, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:26:05+09:00", "last_error_summary": "[h264 @ 0x55b73a5e5240] error while decoding MB 40 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:26:05+09:00", "last_error_summary": "[h264 @ 0x55b73a5e5240] error while decoding MB 40 37, bytestream -5", "last_exit_at": "2026-07-06T12:26:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.069, "last_started_at": "2026-07-06T12:24:04+09:00", "pid": 5946, "process": {"exists": false, "pid": 5946}, "restart_count": 25, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.03, "sample_count": 1540, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18199, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:26:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T12:26:05+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T12:26:05+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T12:26:05.723419+09:00", "mtime_epoch": 1783308365.723419, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:26:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1165.09, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.382, "pushed": 5.382, "read": 5.349}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.462, "count": 160, "max_ms": 16.406, "p95_ms": 12.426}, "gstreamer_push": {"avg_ms": 0.932, "count": 161, "max_ms": 1.263, "p95_ms": 1.077}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.562, "count": 160, "max_ms": 87.973, "p95_ms": 76.182}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.501, "count": 160, "max_ms": 132.947, "p95_ms": 107.444}, "yolo11_640_fire_11class": {"avg_ms": 57.395, "count": 160, "max_ms": 88.649, "p95_ms": 83.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.962, "count": 160, "max_ms": 116.181, "p95_ms": 103.506}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.602, "count": 161, "max_ms": 244.921, "p95_ms": 224.506}, "read_interval": {"avg_ms": 187.251, "count": 160, "max_ms": 254.233, "p95_ms": 233.196}, "triton_wait": {"avg_ms": 76.758, "count": 160, "max_ms": 132.961, "p95_ms": 107.585}, "visualize": {"avg_ms": 7.466, "count": 160, "max_ms": 9.613, "p95_ms": 9.066}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4829.449, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:26:05+09:00", "last_error_summary": "[h264 @ 0x55b73a5e5240] error while decoding MB 40 37, bytestream -5", "last_exit_at": "2026-07-06T12:26:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.074, "last_started_at": "2026-07-06T12:24:04+09:00", "pid": 5946, "process": {"exists": false, "pid": 5946}, "restart_count": 25, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.03, "sample_count": 1540, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18199, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:26:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T12:26:05+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-06T12:26:05+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-06T12:26:05.723419+09:00", "mtime_epoch": 1783308365.723419, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:26:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1165.09, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.381, "pushed": 5.381, "read": 5.348}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.462, "count": 160, "max_ms": 16.406, "p95_ms": 12.426}, "gstreamer_push": {"avg_ms": 0.932, "count": 161, "max_ms": 1.263, "p95_ms": 1.077}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.562, "count": 160, "max_ms": 87.973, "p95_ms": 76.182}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.501, "count": 160, "max_ms": 132.947, "p95_ms": 107.444}, "yolo11_640_fire_11class": {"avg_ms": 57.395, "count": 160, "max_ms": 88.649, "p95_ms": 83.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.962, "count": 160, "max_ms": 116.181, "p95_ms": 103.506}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.602, "count": 161, "max_ms": 244.921, "p95_ms": 224.506}, "read_interval": {"avg_ms": 187.251, "count": 160, "max_ms": 254.233, "p95_ms": 233.196}, "triton_wait": {"avg_ms": 76.758, "count": 160, "max_ms": 132.961, "p95_ms": 107.585}, "visualize": {"avg_ms": 7.466, "count": 160, "max_ms": 9.613, "p95_ms": 9.066}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4945.668, "limit_mb": 8796093022207.996}, "detail": {"pid": 6053, "restart_count": 26, "start_count": 27}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:26:05+09:00", "last_error_summary": "[h264 @ 0x55b73a5e5240] error while decoding MB 40 37, bytestream -5", "last_exit_at": "2026-07-06T12:26:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:26:08+09:00", "pid": 6053, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.707, "name": "ffmpeg", "pid": 6053, "rss_mb": 141.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 26, "restart_count_per_hour": 25, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.226, "sample_count": 1517, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18199, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:26:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.416, "last_frame_pushed_at": "2026-07-06T12:26:05+09:00", "last_frame_read_age_sec": 4.563, "last_frame_read_at": "2026-07-06T12:26:05+09:00", "mjpeg": {"age_sec": 4.394, "exists": true, "mtime": "2026-07-06T12:26:05.883421+09:00", "mtime_epoch": 1783308365.8834214, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193998}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:26:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1165.09, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.611, "pushed": 4.611, "read": 4.577}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.394, "count": 137, "max_ms": 16.406, "p95_ms": 12.532}, "gstreamer_push": {"avg_ms": 0.937, "count": 138, "max_ms": 1.263, "p95_ms": 1.106}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.653, "count": 137, "max_ms": 87.973, "p95_ms": 80.782}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.006, "count": 137, "max_ms": 132.947, "p95_ms": 107.569}, "yolo11_640_fire_11class": {"avg_ms": 57.43, "count": 137, "max_ms": 88.649, "p95_ms": 83.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.212, "count": 137, "max_ms": 116.181, "p95_ms": 105.506}}, "truncated_count": 0}, "processing_total": {"avg_ms": 178.813, "count": 138, "max_ms": 244.921, "p95_ms": 227.125}, "read_interval": {"avg_ms": 186.551, "count": 137, "max_ms": 254.233, "p95_ms": 233.368}, "triton_wait": {"avg_ms": 77.429, "count": 137, "max_ms": 132.961, "p95_ms": 108.173}, "visualize": {"avg_ms": 7.39, "count": 137, "max_ms": 9.613, "p95_ms": 9.066}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 159}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4835.301, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5569d1630580] cabac decode of qscale diff failed at 22 44 [h264 @ 0x5569d1630580] error while decoding MB 22 44, bytestream 4769 [h264 @ 0x5569d1644680] left block unavailable for requested intra mode [h264 @ 0x5569d1644680] error while decoding MB 0 27, bytestream 4054 [h264 @ 0x5569d156e080] left block unavailable for requested intra mode [h264 @ 0x5569d156e080] error while decoding MB 0 41, bytestream 4627 [h264 @ 0x5569d15844c0] cabac decode of qscale diff failed at 35 33 [h264 @ 0", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:28:23+09:00", "last_error_summary": "[h264 @ 0x5569d1630580] cabac decode of qscale diff failed at 22 44 [h264 @ 0x5569d1630580] error while decoding MB 22 44, bytestream 4769 [h264 @ 0x5569d1644680] left block unavailable for requested intra mode [h264 @ 0x5569d1644680] error while decoding MB 0 27, bytestream 4054 [h264 @ 0x5569d156e080] left block unavailable for requested intra mode [h264 @ 0x5569d156e080] error while decoding MB 0 41, bytestream 4627 [h264 @ 0x5569d15844c0] cabac decode of qscale diff failed at 35 33 [h264 @ 0", "last_exit_at": "2026-07-06T12:26:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.147, "last_started_at": "2026-07-06T12:26:08+09:00", "pid": 6053, "process": {"exists": false, "pid": 6053}, "restart_count": 26, "restart_count_per_hour": 24, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.401, "sample_count": 1783, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19150, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:28:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-06T12:28:23+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-06T12:28:23+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T12:28:23.373458+09:00", "mtime_epoch": 1783308503.373458, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:28:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1165.605, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 108, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.607, "pushed": 3.607, "read": 3.607}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.475, "count": 108, "max_ms": 8.035, "p95_ms": 6.792}, "gstreamer_push": {"avg_ms": 1.326, "count": 108, "max_ms": 1.528, "p95_ms": 1.451}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.292, "count": 108, "max_ms": 81.445, "p95_ms": 74.012}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.083, "count": 108, "max_ms": 119.399, "p95_ms": 106.515}, "yolo11_640_fire_11class": {"avg_ms": 59.141, "count": 108, "max_ms": 90.355, "p95_ms": 79.717}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.984, "count": 108, "max_ms": 103.021, "p95_ms": 97.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.299, "count": 108, "max_ms": 132.736, "p95_ms": 120.803}, "read_interval": {"avg_ms": 277.397, "count": 108, "max_ms": 10129.196, "p95_ms": 135.861}, "triton_wait": {"avg_ms": 74.102, "count": 108, "max_ms": 119.414, "p95_ms": 106.529}, "visualize": {"avg_ms": 3.686, "count": 108, "max_ms": 4.869, "p95_ms": 4.062}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 159}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4834.691, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:28:23+09:00", "last_error_summary": "[h264 @ 0x5569d1630580] cabac decode of qscale diff failed at 22 44 [h264 @ 0x5569d1630580] error while decoding MB 22 44, bytestream 4769 [h264 @ 0x5569d1644680] left block unavailable for requested intra mode [h264 @ 0x5569d1644680] error while decoding MB 0 27, bytestream 4054 [h264 @ 0x5569d156e080] left block unavailable for requested intra mode [h264 @ 0x5569d156e080] error while decoding MB 0 41, bytestream 4627 [h264 @ 0x5569d15844c0] cabac decode of qscale diff failed at 35 33 [h264 @ 0", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:28:23+09:00", "last_error_summary": "[h264 @ 0x5569d1630580] cabac decode of qscale diff failed at 22 44 [h264 @ 0x5569d1630580] error while decoding MB 22 44, bytestream 4769 [h264 @ 0x5569d1644680] left block unavailable for requested intra mode [h264 @ 0x5569d1644680] error while decoding MB 0 27, bytestream 4054 [h264 @ 0x5569d156e080] left block unavailable for requested intra mode [h264 @ 0x5569d156e080] error while decoding MB 0 41, bytestream 4627 [h264 @ 0x5569d15844c0] cabac decode of qscale diff failed at 35 33 [h264 @ 0", "last_exit_at": "2026-07-06T12:28:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.182, "last_started_at": "2026-07-06T12:26:08+09:00", "pid": 6053, "process": {"exists": false, "pid": 6053}, "restart_count": 26, "restart_count_per_hour": 24, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.401, "sample_count": 1784, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19151, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:28:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T12:28:23+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-06T12:28:23+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-06T12:28:23.373458+09:00", "mtime_epoch": 1783308503.373458, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:28:23+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1165.609, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.637, "pushed": 3.637, "read": 3.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.473, "count": 109, "max_ms": 8.035, "p95_ms": 6.792}, "gstreamer_push": {"avg_ms": 1.325, "count": 109, "max_ms": 1.528, "p95_ms": 1.451}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.273, "count": 109, "max_ms": 81.445, "p95_ms": 74.012}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.065, "count": 109, "max_ms": 119.399, "p95_ms": 106.515}, "yolo11_640_fire_11class": {"avg_ms": 59.183, "count": 109, "max_ms": 90.355, "p95_ms": 79.717}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.997, "count": 109, "max_ms": 103.021, "p95_ms": 97.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.276, "count": 109, "max_ms": 132.736, "p95_ms": 120.803}, "read_interval": {"avg_ms": 277.397, "count": 108, "max_ms": 10129.196, "p95_ms": 135.861}, "triton_wait": {"avg_ms": 74.084, "count": 109, "max_ms": 119.414, "p95_ms": 106.529}, "visualize": {"avg_ms": 3.686, "count": 109, "max_ms": 4.869, "p95_ms": 4.062}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 159}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4834.695, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:28:23+09:00", "last_error_summary": "[h264 @ 0x5569d1630580] cabac decode of qscale diff failed at 22 44 [h264 @ 0x5569d1630580] error while decoding MB 22 44, bytestream 4769 [h264 @ 0x5569d1644680] left block unavailable for requested intra mode [h264 @ 0x5569d1644680] error while decoding MB 0 27, bytestream 4054 [h264 @ 0x5569d156e080] left block unavailable for requested intra mode [h264 @ 0x5569d156e080] error while decoding MB 0 41, bytestream 4627 [h264 @ 0x5569d15844c0] cabac decode of qscale diff failed at 35 33 [h264 @ 0", "last_exit_at": "2026-07-06T12:28:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.189, "last_started_at": "2026-07-06T12:26:08+09:00", "pid": 6053, "process": {"exists": false, "pid": 6053}, "restart_count": 26, "restart_count_per_hour": 24, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.401, "sample_count": 1784, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19151, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:28:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T12:28:23+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-06T12:28:23+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-06T12:28:23.373458+09:00", "mtime_epoch": 1783308503.373458, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:28:23+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1165.609, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.636, "pushed": 3.636, "read": 3.602}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.473, "count": 109, "max_ms": 8.035, "p95_ms": 6.792}, "gstreamer_push": {"avg_ms": 1.325, "count": 109, "max_ms": 1.528, "p95_ms": 1.451}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.273, "count": 109, "max_ms": 81.445, "p95_ms": 74.012}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.065, "count": 109, "max_ms": 119.399, "p95_ms": 106.515}, "yolo11_640_fire_11class": {"avg_ms": 59.183, "count": 109, "max_ms": 90.355, "p95_ms": 79.717}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.997, "count": 109, "max_ms": 103.021, "p95_ms": 97.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.276, "count": 109, "max_ms": 132.736, "p95_ms": 120.803}, "read_interval": {"avg_ms": 277.397, "count": 108, "max_ms": 10129.196, "p95_ms": 135.861}, "triton_wait": {"avg_ms": 74.084, "count": 109, "max_ms": 119.414, "p95_ms": 106.529}, "visualize": {"avg_ms": 3.686, "count": 109, "max_ms": 4.869, "p95_ms": 4.062}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 159}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4942.699, "limit_mb": 8796093022207.996}, "detail": {"pid": 6160, "restart_count": 27, "start_count": 28}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:28:23+09:00", "last_error_summary": "[h264 @ 0x5569d1630580] cabac decode of qscale diff failed at 22 44 [h264 @ 0x5569d1630580] error while decoding MB 22 44, bytestream 4769 [h264 @ 0x5569d1644680] left block unavailable for requested intra mode [h264 @ 0x5569d1644680] error while decoding MB 0 27, bytestream 4054 [h264 @ 0x5569d156e080] left block unavailable for requested intra mode [h264 @ 0x5569d156e080] error while decoding MB 0 41, bytestream 4627 [h264 @ 0x5569d15844c0] cabac decode of qscale diff failed at 35 33 [h264 @ 0", "last_exit_at": "2026-07-06T12:28:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:28:25+09:00", "pid": 6160, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.922, "name": "ffmpeg", "pid": 6160, "rss_mb": 141.922, "state": "S (sleeping)", "threads": 106}, "restart_count": 27, "restart_count_per_hour": 25, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.46, "sample_count": 1756, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19151, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:28:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.325, "last_frame_pushed_at": "2026-07-06T12:28:23+09:00", "last_frame_read_age_sec": 4.413, "last_frame_read_at": "2026-07-06T12:28:23+09:00", "mjpeg": {"age_sec": 4.309, "exists": true, "mtime": "2026-07-06T12:28:23.469460+09:00", "mtime_epoch": 1783308503.4694595, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:28:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1165.609, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.171, "pushed": 2.171, "read": 2.138}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.444, "count": 65, "max_ms": 8.035, "p95_ms": 6.826}, "gstreamer_push": {"avg_ms": 1.33, "count": 65, "max_ms": 1.477, "p95_ms": 1.446}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.789, "count": 65, "max_ms": 81.445, "p95_ms": 75.005}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.341, "count": 65, "max_ms": 119.399, "p95_ms": 110.932}, "yolo11_640_fire_11class": {"avg_ms": 59.595, "count": 65, "max_ms": 90.355, "p95_ms": 83.405}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.28, "count": 65, "max_ms": 103.021, "p95_ms": 98.135}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.489, "count": 65, "max_ms": 132.736, "p95_ms": 123.588}, "read_interval": {"avg_ms": 400.057, "count": 64, "max_ms": 10129.196, "p95_ms": 142.177}, "triton_wait": {"avg_ms": 75.354, "count": 65, "max_ms": 119.414, "p95_ms": 110.945}, "visualize": {"avg_ms": 3.669, "count": 65, "max_ms": 4.644, "p95_ms": 3.957}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 287}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4782.598, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55afaa07e140] error while decoding MB 15 43, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:30:48+09:00", "last_error_summary": "[h264 @ 0x55afaa07e140] error while decoding MB 15 43, bytestream -7", "last_exit_at": "2026-07-06T12:18:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 751.513, "last_started_at": "2026-07-06T12:18:16+09:00", "pid": 5520, "process": {"exists": false, "pid": 5520}, "restart_count": 8, "restart_count_per_hour": 7, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.193, "sample_count": 2125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:30:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T12:30:48+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-06T12:30:48+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T12:30:48.055649+09:00", "mtime_epoch": 1783308648.055649, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226866}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:30:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1072.289, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.192, "pushed": 7.192, "read": 7.192}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.091, "count": 215, "max_ms": 9.611, "p95_ms": 7.675}, "gstreamer_push": {"avg_ms": 1.311, "count": 215, "max_ms": 4.421, "p95_ms": 2.621}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.525, "count": 215, "max_ms": 87.871, "p95_ms": 78.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.57, "count": 215, "max_ms": 122.281, "p95_ms": 108.61}, "yolo11_640_fire_11class": {"avg_ms": 64.884, "count": 215, "max_ms": 94.733, "p95_ms": 84.848}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.796, "count": 215, "max_ms": 104.196, "p95_ms": 99.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.261, "count": 215, "max_ms": 182.949, "p95_ms": 164.031}, "read_interval": {"avg_ms": 139.419, "count": 215, "max_ms": 193.206, "p95_ms": 174.89}, "triton_wait": {"avg_ms": 79.268, "count": 215, "max_ms": 122.295, "p95_ms": 108.62}, "visualize": {"avg_ms": 5.037, "count": 215, "max_ms": 7.145, "p95_ms": 5.768}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 287}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4781.871, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:30:48+09:00", "last_error_summary": "[h264 @ 0x55afaa07e140] error while decoding MB 15 43, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:30:48+09:00", "last_error_summary": "[h264 @ 0x55afaa07e140] error while decoding MB 15 43, bytestream -7", "last_exit_at": "2026-07-06T12:30:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 751.585, "last_started_at": "2026-07-06T12:18:16+09:00", "pid": 5520, "process": {"exists": false, "pid": 5520}, "restart_count": 8, "restart_count_per_hour": 7, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.193, "sample_count": 2126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34457, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:30:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T12:30:48+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-06T12:30:48+09:00", "mjpeg": {"age_sec": 0.129, "exists": true, "mtime": "2026-07-06T12:30:48.055649+09:00", "mtime_epoch": 1783308648.055649, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226866}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:30:48+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1072.289, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.208, "pushed": 7.208, "read": 7.174}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.125, "count": 215, "max_ms": 13.297, "p95_ms": 7.699}, "gstreamer_push": {"avg_ms": 1.31, "count": 216, "max_ms": 4.421, "p95_ms": 2.621}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.389, "count": 215, "max_ms": 87.871, "p95_ms": 78.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.441, "count": 215, "max_ms": 122.281, "p95_ms": 108.61}, "yolo11_640_fire_11class": {"avg_ms": 64.82, "count": 215, "max_ms": 94.733, "p95_ms": 84.848}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.675, "count": 215, "max_ms": 104.196, "p95_ms": 99.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.274, "count": 216, "max_ms": 182.949, "p95_ms": 164.031}, "read_interval": {"avg_ms": 139.419, "count": 215, "max_ms": 193.206, "p95_ms": 174.89}, "triton_wait": {"avg_ms": 79.138, "count": 215, "max_ms": 122.295, "p95_ms": 108.62}, "visualize": {"avg_ms": 5.05, "count": 215, "max_ms": 7.852, "p95_ms": 5.814}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 287}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4781.988, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:30:48+09:00", "last_error_summary": "[h264 @ 0x55afaa07e140] error while decoding MB 15 43, bytestream -7", "last_exit_at": "2026-07-06T12:30:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 751.591, "last_started_at": "2026-07-06T12:18:16+09:00", "pid": 5520, "process": {"exists": false, "pid": 5520}, "restart_count": 8, "restart_count_per_hour": 7, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.193, "sample_count": 2126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34457, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:30:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T12:30:48+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-06T12:30:48+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-06T12:30:48.055649+09:00", "mtime_epoch": 1783308648.055649, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226866}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:30:48+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1072.289, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.206, "pushed": 7.206, "read": 7.173}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.125, "count": 215, "max_ms": 13.297, "p95_ms": 7.699}, "gstreamer_push": {"avg_ms": 1.31, "count": 216, "max_ms": 4.421, "p95_ms": 2.621}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.389, "count": 215, "max_ms": 87.871, "p95_ms": 78.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.441, "count": 215, "max_ms": 122.281, "p95_ms": 108.61}, "yolo11_640_fire_11class": {"avg_ms": 64.82, "count": 215, "max_ms": 94.733, "p95_ms": 84.848}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.675, "count": 215, "max_ms": 104.196, "p95_ms": 99.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.274, "count": 216, "max_ms": 182.949, "p95_ms": 164.031}, "read_interval": {"avg_ms": 139.419, "count": 215, "max_ms": 193.206, "p95_ms": 174.89}, "triton_wait": {"avg_ms": 79.138, "count": 215, "max_ms": 122.295, "p95_ms": 108.62}, "visualize": {"avg_ms": 5.05, "count": 215, "max_ms": 7.852, "p95_ms": 5.814}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_111800_6db20b81.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 287}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4898.98, "limit_mb": 8796093022207.996}, "detail": {"pid": 6267, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:30:48+09:00", "last_error_summary": "[h264 @ 0x55afaa07e140] error while decoding MB 15 43, bytestream -7", "last_exit_at": "2026-07-06T12:30:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:30:50+09:00", "pid": 6267, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.504, "name": "ffmpeg", "pid": 6267, "rss_mb": 149.504, "state": "R (running)", "threads": 49}, "restart_count": 9, "restart_count_per_hour": 8, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.193, "sample_count": 2094, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34457, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:30:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6db20b81", "last_frame_pushed_age_sec": 4.517, "last_frame_pushed_at": "2026-07-06T12:30:48+09:00", "last_frame_read_age_sec": 4.651, "last_frame_read_at": "2026-07-06T12:30:48+09:00", "mjpeg": {"age_sec": 4.498, "exists": true, "mtime": "2026-07-06T12:30:48.195651+09:00", "mtime_epoch": 1783308648.195651, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222089}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:30:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1072.289, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.149, "pushed": 6.149, "read": 6.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.106, "count": 184, "max_ms": 13.297, "p95_ms": 7.699}, "gstreamer_push": {"avg_ms": 1.318, "count": 184, "max_ms": 4.421, "p95_ms": 2.621}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.5, "count": 184, "max_ms": 87.871, "p95_ms": 78.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.358, "count": 184, "max_ms": 122.281, "p95_ms": 108.913}, "yolo11_640_fire_11class": {"avg_ms": 65.031, "count": 184, "max_ms": 94.733, "p95_ms": 85.344}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.995, "count": 184, "max_ms": 104.196, "p95_ms": 99.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.57, "count": 184, "max_ms": 182.949, "p95_ms": 164.078}, "read_interval": {"avg_ms": 138.724, "count": 183, "max_ms": 193.206, "p95_ms": 174.926}, "triton_wait": {"avg_ms": 79.143, "count": 184, "max_ms": 122.295, "p95_ms": 108.927}, "visualize": {"avg_ms": 5.03, "count": 184, "max_ms": 7.852, "p95_ms": 5.814}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-06T11:18:00+09:00", "state": "running", "task_id": "ce642936-0e00-4300-990e-1afc01d84271", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4839.273, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c932c96100] error while decoding MB 27 43, bytestream -5 [h264 @ 0x55c932b4b000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c932b4b000] error while decoding MB 0 42, bytestream 73417 [h264 @ 0x55c933680600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c933680600] error while decoding MB 0 20, bytestream 4377 [h264 @ 0x55c932c59d40] error while decoding MB 68 37, bytestream -9 [in#0/rtsp @ 0x55c932b30e00] Error during demuxing: Connection ti", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:35:03+09:00", "last_error_summary": "[h264 @ 0x55c932c96100] error while decoding MB 27 43, bytestream -5 [h264 @ 0x55c932b4b000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c932b4b000] error while decoding MB 0 42, bytestream 73417 [h264 @ 0x55c933680600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c933680600] error while decoding MB 0 20, bytestream 4377 [h264 @ 0x55c932c59d40] error while decoding MB 68 37, bytestream -9 [in#0/rtsp @ 0x55c932b30e00] Error during demuxing: Connection ti", "last_exit_at": "2026-07-06T12:28:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 397.378, "last_started_at": "2026-07-06T12:28:25+09:00", "pid": 6160, "process": {"exists": false, "pid": 6160}, "restart_count": 27, "restart_count_per_hour": 22, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.72, "sample_count": 2662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22680, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:35:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T12:35:03+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T12:35:03+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T12:35:03.115597+09:00", "mtime_epoch": 1783308903.1155965, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:35:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1167.156, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 85, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.8, "pushed": 2.8, "read": 2.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.093, "count": 84, "max_ms": 11.858, "p95_ms": 9.529}, "gstreamer_push": {"avg_ms": 1.164, "count": 84, "max_ms": 2.182, "p95_ms": 1.336}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.712, "count": 84, "max_ms": 91.238, "p95_ms": 76.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.95, "count": 84, "max_ms": 127.544, "p95_ms": 113.226}, "yolo11_640_fire_11class": {"avg_ms": 61.927, "count": 84, "max_ms": 95.936, "p95_ms": 86.416}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.471, "count": 84, "max_ms": 110.1, "p95_ms": 101.587}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.628, "count": 84, "max_ms": 182.232, "p95_ms": 166.64}, "read_interval": {"avg_ms": 354.141, "count": 85, "max_ms": 19111.079, "p95_ms": 183.853}, "triton_wait": {"avg_ms": 76.334, "count": 84, "max_ms": 127.557, "p95_ms": 113.24}, "visualize": {"avg_ms": 4.662, "count": 84, "max_ms": 6.814, "p95_ms": 6.11}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4838.855, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:35:03+09:00", "last_error_summary": "[h264 @ 0x55c932c96100] error while decoding MB 27 43, bytestream -5 [h264 @ 0x55c932b4b000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c932b4b000] error while decoding MB 0 42, bytestream 73417 [h264 @ 0x55c933680600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c933680600] error while decoding MB 0 20, bytestream 4377 [h264 @ 0x55c932c59d40] error while decoding MB 68 37, bytestream -9 [in#0/rtsp @ 0x55c932b30e00] Error during demuxing: Connection ti", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:35:03+09:00", "last_error_summary": "[h264 @ 0x55c932c96100] error while decoding MB 27 43, bytestream -5 [h264 @ 0x55c932b4b000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c932b4b000] error while decoding MB 0 42, bytestream 73417 [h264 @ 0x55c933680600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c933680600] error while decoding MB 0 20, bytestream 4377 [h264 @ 0x55c932c59d40] error while decoding MB 68 37, bytestream -9 [in#0/rtsp @ 0x55c932b30e00] Error during demuxing: Connection ti", "last_exit_at": "2026-07-06T12:35:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 397.427, "last_started_at": "2026-07-06T12:28:25+09:00", "pid": 6160, "process": {"exists": false, "pid": 6160}, "restart_count": 27, "restart_count_per_hour": 22, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.72, "sample_count": 2662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22681, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:35:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T12:35:03+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-06T12:35:03+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-06T12:35:03.115597+09:00", "mtime_epoch": 1783308903.1155965, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:35:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1167.156, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.84, "pushed": 2.84, "read": 2.806}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.104, "count": 85, "max_ms": 11.858, "p95_ms": 9.529}, "gstreamer_push": {"avg_ms": 1.166, "count": 85, "max_ms": 2.182, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.463, "count": 85, "max_ms": 91.238, "p95_ms": 76.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.902, "count": 85, "max_ms": 127.544, "p95_ms": 113.226}, "yolo11_640_fire_11class": {"avg_ms": 61.752, "count": 85, "max_ms": 95.936, "p95_ms": 86.416}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.241, "count": 85, "max_ms": 110.1, "p95_ms": 101.587}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.281, "count": 85, "max_ms": 182.232, "p95_ms": 166.64}, "read_interval": {"avg_ms": 356.494, "count": 84, "max_ms": 19111.079, "p95_ms": 183.853}, "triton_wait": {"avg_ms": 76.282, "count": 85, "max_ms": 127.557, "p95_ms": 113.24}, "visualize": {"avg_ms": 4.657, "count": 85, "max_ms": 6.814, "p95_ms": 6.11}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4838.977, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:35:03+09:00", "last_error_summary": "[h264 @ 0x55c932c96100] error while decoding MB 27 43, bytestream -5 [h264 @ 0x55c932b4b000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c932b4b000] error while decoding MB 0 42, bytestream 73417 [h264 @ 0x55c933680600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c933680600] error while decoding MB 0 20, bytestream 4377 [h264 @ 0x55c932c59d40] error while decoding MB 68 37, bytestream -9 [in#0/rtsp @ 0x55c932b30e00] Error during demuxing: Connection ti", "last_exit_at": "2026-07-06T12:35:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 397.436, "last_started_at": "2026-07-06T12:28:25+09:00", "pid": 6160, "process": {"exists": false, "pid": 6160}, "restart_count": 27, "restart_count_per_hour": 22, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.72, "sample_count": 2662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22681, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:35:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T12:35:03+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T12:35:03+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-06T12:35:03.115597+09:00", "mtime_epoch": 1783308903.1155965, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:35:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1167.156, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.839, "pushed": 2.839, "read": 2.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.104, "count": 85, "max_ms": 11.858, "p95_ms": 9.529}, "gstreamer_push": {"avg_ms": 1.166, "count": 85, "max_ms": 2.182, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.463, "count": 85, "max_ms": 91.238, "p95_ms": 76.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.902, "count": 85, "max_ms": 127.544, "p95_ms": 113.226}, "yolo11_640_fire_11class": {"avg_ms": 61.752, "count": 85, "max_ms": 95.936, "p95_ms": 86.416}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.241, "count": 85, "max_ms": 110.1, "p95_ms": 101.587}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.281, "count": 85, "max_ms": 182.232, "p95_ms": 166.64}, "read_interval": {"avg_ms": 356.494, "count": 84, "max_ms": 19111.079, "p95_ms": 183.853}, "triton_wait": {"avg_ms": 76.282, "count": 85, "max_ms": 127.557, "p95_ms": 113.24}, "visualize": {"avg_ms": 4.657, "count": 85, "max_ms": 6.814, "p95_ms": 6.11}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4946.93, "limit_mb": 8796093022207.996}, "detail": {"pid": 6419, "restart_count": 28, "start_count": 29}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:35:03+09:00", "last_error_summary": "[h264 @ 0x55c932c96100] error while decoding MB 27 43, bytestream -5 [h264 @ 0x55c932b4b000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c932b4b000] error while decoding MB 0 42, bytestream 73417 [h264 @ 0x55c933680600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c933680600] error while decoding MB 0 20, bytestream 4377 [h264 @ 0x55c932c59d40] error while decoding MB 68 37, bytestream -9 [in#0/rtsp @ 0x55c932b30e00] Error during demuxing: Connection ti", "last_exit_at": "2026-07-06T12:35:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:35:05+09:00", "pid": 6419, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.637, "name": "ffmpeg", "pid": 6419, "rss_mb": 141.637, "state": "S (sleeping)", "threads": 106}, "restart_count": 28, "restart_count_per_hour": 23, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.961, "sample_count": 2617, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22681, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:35:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 4.427, "last_frame_pushed_at": "2026-07-06T12:35:03+09:00", "last_frame_read_age_sec": 4.518, "last_frame_read_at": "2026-07-06T12:35:03+09:00", "mjpeg": {"age_sec": 4.408, "exists": true, "mtime": "2026-07-06T12:35:03.215598+09:00", "mtime_epoch": 1783308903.215598, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185342}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:35:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1167.156, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.772, "pushed": 1.772, "read": 1.738}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.639, "count": 53, "max_ms": 10.597, "p95_ms": 8.745}, "gstreamer_push": {"avg_ms": 1.208, "count": 53, "max_ms": 2.182, "p95_ms": 1.38}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.328, "count": 53, "max_ms": 91.238, "p95_ms": 79.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.772, "count": 53, "max_ms": 127.544, "p95_ms": 119.497}, "yolo11_640_fire_11class": {"avg_ms": 62.926, "count": 53, "max_ms": 95.936, "p95_ms": 89.592}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.355, "count": 53, "max_ms": 110.1, "p95_ms": 105.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.902, "count": 53, "max_ms": 182.232, "p95_ms": 178.77}, "read_interval": {"avg_ms": 491.868, "count": 52, "max_ms": 19111.079, "p95_ms": 192.548}, "triton_wait": {"avg_ms": 78.784, "count": 53, "max_ms": 127.557, "p95_ms": 119.51}, "visualize": {"avg_ms": 4.416, "count": 53, "max_ms": 6.814, "p95_ms": 5.986}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_111801_20966a89.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 220}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4528.168, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560f45130040] left block unavailable for requested intra mode [h264 @ 0x560f45130040] error while decoding MB 0 40, bytestream 8292", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:42:04+09:00", "last_error_summary": "[h264 @ 0x560f45130040] left block unavailable for requested intra mode [h264 @ 0x560f45130040] error while decoding MB 0 40, bytestream 8292", "last_exit_at": "2026-07-06T12:35:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 418.57, "last_started_at": "2026-07-06T12:35:05+09:00", "pid": 6419, "process": {"exists": false, "pid": 6419}, "restart_count": 28, "restart_count_per_hour": 20, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.095, "sample_count": 2598, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26450, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:42:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "20966a89", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T12:42:04+09:00", "last_frame_read_age_sec": 0.173, "last_frame_read_at": "2026-07-06T12:42:04+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T12:42:04.142255+09:00", "mtime_epoch": 1783309324.1422546, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:42:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1169.059, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.74, "pushed": 5.74, "read": 5.707}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.985, "count": 171, "max_ms": 13.71, "p95_ms": 10.463}, "gstreamer_push": {"avg_ms": 0.999, "count": 172, "max_ms": 1.312, "p95_ms": 1.148}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.241, "count": 171, "max_ms": 80.175, "p95_ms": 71.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.722, "count": 171, "max_ms": 129.489, "p95_ms": 108.323}, "yolo11_640_fire_11class": {"avg_ms": 61.246, "count": 171, "max_ms": 87.818, "p95_ms": 79.21}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.599, "count": 171, "max_ms": 109.379, "p95_ms": 98.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 165.555, "count": 172, "max_ms": 211.246, "p95_ms": 194.673}, "read_interval": {"avg_ms": 175.285, "count": 171, "max_ms": 248.402, "p95_ms": 206.346}, "triton_wait": {"avg_ms": 84.939, "count": 171, "max_ms": 129.502, "p95_ms": 108.336}, "visualize": {"avg_ms": 6.256, "count": 171, "max_ms": 8.104, "p95_ms": 7.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-06T11:18:01+09:00", "state": "running", "task_id": "7a8d317e-d87f-4c7c-bc69-3c2897604379", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4532.434, "limit_mb": 8796093022207.996}, "detail": {"pid": 6527, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:42:03+09:00", "pid": 6527, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.547, "name": "ffmpeg", "pid": 6527, "rss_mb": 57.547, "state": "S (sleeping)", "threads": 1}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.925}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 2.745, "exists": true, "mtime": "2026-07-06T12:42:02.614230+09:00", "mtime_epoch": 1783309322.6142302, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 204143}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:42:05+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1037.719, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4876.828, "limit_mb": 8796093022207.996}, "detail": {"pid": 6530, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:42:04+09:00", "pid": 6530, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.562, "name": "ffmpeg", "pid": 6530, "rss_mb": 139.562, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.932}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 2.801, "exists": true, "mtime": "2026-07-06T12:42:04.142255+09:00", "mtime_epoch": 1783309324.1422546, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:42:06+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1163.266, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5364.145, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561a62ee6600] error while decoding MB 69 31, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:44:16+09:00", "last_error_summary": "[h264 @ 0x561a62ee6600] error while decoding MB 69 31, bytestream -17", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 131.511, "last_started_at": "2026-07-06T12:42:04+09:00", "pid": 6530, "process": {"exists": false, "pid": 6530}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 199.663, "sample_count": 746, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 168.419}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:44:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T12:44:16+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T12:44:16+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T12:44:16.410473+09:00", "mtime_epoch": 1783309456.4104733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188469}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:44:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1444.121, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 172, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.747, "pushed": 5.747, "read": 5.747}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.1, "count": 171, "max_ms": 11.645, "p95_ms": 9.393}, "gstreamer_push": {"avg_ms": 1.051, "count": 172, "max_ms": 3.457, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.892, "count": 171, "max_ms": 85.495, "p95_ms": 73.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.985, "count": 171, "max_ms": 116.884, "p95_ms": 106.453}, "yolo11_640_fire_11class": {"avg_ms": 61.183, "count": 171, "max_ms": 88.086, "p95_ms": 82.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.571, "count": 171, "max_ms": 106.521, "p95_ms": 97.729}}, "truncated_count": 0}, "processing_total": {"avg_ms": 167.296, "count": 172, "max_ms": 233.056, "p95_ms": 200.637}, "read_interval": {"avg_ms": 174.676, "count": 172, "max_ms": 242.84, "p95_ms": 207.866}, "triton_wait": {"avg_ms": 80.542, "count": 171, "max_ms": 116.896, "p95_ms": 106.465}, "visualize": {"avg_ms": 6.669, "count": 171, "max_ms": 9.397, "p95_ms": 8.145}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5360.094, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:44:16+09:00", "last_error_summary": "[h264 @ 0x561a62ee6600] error while decoding MB 69 31, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:44:16+09:00", "last_error_summary": "[h264 @ 0x561a62ee6600] error while decoding MB 69 31, bytestream -17", "last_exit_at": "2026-07-06T12:44:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.616, "last_started_at": "2026-07-06T12:42:04+09:00", "pid": 6530, "process": {"exists": false, "pid": 6530}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 199.663, "sample_count": 747, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 168.317}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 747, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:44:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T12:44:16+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-06T12:44:16+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-06T12:44:16.410473+09:00", "mtime_epoch": 1783309456.4104733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188469}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:44:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1440.516, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.761, "pushed": 5.761, "read": 5.727}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.125, "count": 172, "max_ms": 11.645, "p95_ms": 9.425}, "gstreamer_push": {"avg_ms": 1.066, "count": 172, "max_ms": 3.573, "p95_ms": 1.221}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.814, "count": 172, "max_ms": 85.495, "p95_ms": 73.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.888, "count": 172, "max_ms": 116.884, "p95_ms": 106.453}, "yolo11_640_fire_11class": {"avg_ms": 61.135, "count": 172, "max_ms": 88.086, "p95_ms": 82.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.511, "count": 172, "max_ms": 106.521, "p95_ms": 97.729}}, "truncated_count": 0}, "processing_total": {"avg_ms": 167.174, "count": 172, "max_ms": 233.056, "p95_ms": 200.637}, "read_interval": {"avg_ms": 174.654, "count": 171, "max_ms": 242.84, "p95_ms": 207.866}, "triton_wait": {"avg_ms": 80.442, "count": 172, "max_ms": 116.896, "p95_ms": 106.465}, "visualize": {"avg_ms": 6.666, "count": 172, "max_ms": 9.397, "p95_ms": 8.145}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.858, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5360.098, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:44:16+09:00", "last_error_summary": "[h264 @ 0x561a62ee6600] error while decoding MB 69 31, bytestream -17", "last_exit_at": "2026-07-06T12:44:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.62, "last_started_at": "2026-07-06T12:42:04+09:00", "pid": 6530, "process": {"exists": false, "pid": 6530}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 199.663, "sample_count": 747, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 168.312}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 747, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:44:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T12:44:16+09:00", "last_frame_read_age_sec": 0.164, "last_frame_read_at": "2026-07-06T12:44:16+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-06T12:44:16.410473+09:00", "mtime_epoch": 1783309456.4104733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188469}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:44:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1440.516, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.76, "pushed": 5.76, "read": 5.726}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.125, "count": 172, "max_ms": 11.645, "p95_ms": 9.425}, "gstreamer_push": {"avg_ms": 1.066, "count": 172, "max_ms": 3.573, "p95_ms": 1.221}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.814, "count": 172, "max_ms": 85.495, "p95_ms": 73.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.888, "count": 172, "max_ms": 116.884, "p95_ms": 106.453}, "yolo11_640_fire_11class": {"avg_ms": 61.135, "count": 172, "max_ms": 88.086, "p95_ms": 82.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.511, "count": 172, "max_ms": 106.521, "p95_ms": 97.729}}, "truncated_count": 0}, "processing_total": {"avg_ms": 167.174, "count": 172, "max_ms": 233.056, "p95_ms": 200.637}, "read_interval": {"avg_ms": 174.654, "count": 171, "max_ms": 242.84, "p95_ms": 207.866}, "triton_wait": {"avg_ms": 80.442, "count": 172, "max_ms": 116.896, "p95_ms": 106.465}, "visualize": {"avg_ms": 6.666, "count": 172, "max_ms": 9.397, "p95_ms": 8.145}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5466.934, "limit_mb": 8796093022207.996}, "detail": {"pid": 6789, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:44:16+09:00", "last_error_summary": "[h264 @ 0x561a62ee6600] error while decoding MB 69 31, bytestream -17", "last_exit_at": "2026-07-06T12:44:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:44:18+09:00", "pid": 6789, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.195, "name": "ffmpeg", "pid": 6789, "rss_mb": 139.195, "state": "S (sleeping)", "threads": 106}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 199.663, "sample_count": 747, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 163.959}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 747, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:44:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.361, "last_frame_pushed_at": "2026-07-06T12:44:16+09:00", "last_frame_read_age_sec": 4.516, "last_frame_read_at": "2026-07-06T12:44:16+09:00", "mjpeg": {"age_sec": 4.344, "exists": true, "mtime": "2026-07-06T12:44:16.570478+09:00", "mtime_epoch": 1783309456.5704784, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187959}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:44:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1440.516, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.964, "pushed": 4.964, "read": 4.931}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.928, "count": 148, "max_ms": 11.497, "p95_ms": 9.031}, "gstreamer_push": {"avg_ms": 1.093, "count": 148, "max_ms": 3.573, "p95_ms": 1.811}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.389, "count": 148, "max_ms": 85.495, "p95_ms": 73.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.637, "count": 148, "max_ms": 116.884, "p95_ms": 106.453}, "yolo11_640_fire_11class": {"avg_ms": 61.68, "count": 148, "max_ms": 88.086, "p95_ms": 82.98}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.449, "count": 148, "max_ms": 106.521, "p95_ms": 97.826}}, "truncated_count": 0}, "processing_total": {"avg_ms": 165.787, "count": 148, "max_ms": 233.056, "p95_ms": 200.135}, "read_interval": {"avg_ms": 173.457, "count": 147, "max_ms": 242.84, "p95_ms": 207.866}, "triton_wait": {"avg_ms": 82.172, "count": 148, "max_ms": 116.896, "p95_ms": 106.465}, "visualize": {"avg_ms": 6.483, "count": 148, "max_ms": 8.259, "p95_ms": 7.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.814, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 14}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5365.723, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55de8668e540] error while decoding MB 76 22, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:46:52+09:00", "last_error_summary": "[h264 @ 0x55de8668e540] error while decoding MB 76 22, bytestream -7", "last_exit_at": "2026-07-06T12:44:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.375, "last_started_at": "2026-07-06T12:44:18+09:00", "pid": 6789, "process": {"exists": false, "pid": 6789}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 195.16, "sample_count": 1735, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 12.584}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1735, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:46:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-06T12:46:52+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-06T12:46:52+09:00", "mjpeg": {"age_sec": 0.034, "exists": true, "mtime": "2026-07-06T12:46:52.255288+09:00", "mtime_epoch": 1783309612.2552881, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169967}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:46:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1441.66, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 208, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.947, "pushed": 6.947, "read": 6.947}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.949, "count": 208, "max_ms": 9.262, "p95_ms": 8.343}, "gstreamer_push": {"avg_ms": 1.252, "count": 208, "max_ms": 5.616, "p95_ms": 2.366}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.368, "count": 208, "max_ms": 87.374, "p95_ms": 80.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.467, "count": 208, "max_ms": 112.142, "p95_ms": 101.169}, "yolo11_640_fire_11class": {"avg_ms": 65.31, "count": 208, "max_ms": 89.611, "p95_ms": 83.984}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.421, "count": 208, "max_ms": 108.336, "p95_ms": 101.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.856, "count": 208, "max_ms": 177.574, "p95_ms": 160.24}, "read_interval": {"avg_ms": 144.446, "count": 208, "max_ms": 189.27, "p95_ms": 169.27}, "triton_wait": {"avg_ms": 80.902, "count": 208, "max_ms": 112.156, "p95_ms": 102.472}, "visualize": {"avg_ms": 5.158, "count": 208, "max_ms": 9.335, "p95_ms": 6.028}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 14}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5364.82, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:46:52+09:00", "last_error_summary": "[h264 @ 0x55de8668e540] error while decoding MB 76 22, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:46:52+09:00", "last_error_summary": "[h264 @ 0x55de8668e540] error while decoding MB 76 22, bytestream -7", "last_exit_at": "2026-07-06T12:46:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.447, "last_started_at": "2026-07-06T12:44:18+09:00", "pid": 6789, "process": {"exists": false, "pid": 6789}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 195.16, "sample_count": 1736, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 12.513}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1736, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:46:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T12:46:52+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-06T12:46:52+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T12:46:52.255288+09:00", "mtime_epoch": 1783309612.2552881, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169967}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:46:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1441.66, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.964, "pushed": 6.964, "read": 6.93}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.949, "count": 208, "max_ms": 9.262, "p95_ms": 8.343}, "gstreamer_push": {"avg_ms": 1.251, "count": 208, "max_ms": 5.616, "p95_ms": 2.366}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.189, "count": 208, "max_ms": 87.374, "p95_ms": 80.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.297, "count": 208, "max_ms": 112.142, "p95_ms": 101.169}, "yolo11_640_fire_11class": {"avg_ms": 65.179, "count": 208, "max_ms": 89.611, "p95_ms": 83.984}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.245, "count": 208, "max_ms": 108.336, "p95_ms": 101.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.672, "count": 208, "max_ms": 177.574, "p95_ms": 160.24}, "read_interval": {"avg_ms": 144.367, "count": 207, "max_ms": 189.27, "p95_ms": 169.27}, "triton_wait": {"avg_ms": 80.732, "count": 208, "max_ms": 112.156, "p95_ms": 102.472}, "visualize": {"avg_ms": 5.158, "count": 208, "max_ms": 9.335, "p95_ms": 6.028}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 14}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5364.855, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:46:52+09:00", "last_error_summary": "[h264 @ 0x55de8668e540] error while decoding MB 76 22, bytestream -7", "last_exit_at": "2026-07-06T12:46:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.453, "last_started_at": "2026-07-06T12:44:18+09:00", "pid": 6789, "process": {"exists": false, "pid": 6789}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 195.16, "sample_count": 1736, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 12.508}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1736, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:46:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T12:46:52+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T12:46:52+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T12:46:52.255288+09:00", "mtime_epoch": 1783309612.2552881, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169967}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:46:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1441.66, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.962, "pushed": 6.962, "read": 6.929}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.949, "count": 208, "max_ms": 9.262, "p95_ms": 8.343}, "gstreamer_push": {"avg_ms": 1.251, "count": 208, "max_ms": 5.616, "p95_ms": 2.366}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.189, "count": 208, "max_ms": 87.374, "p95_ms": 80.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.297, "count": 208, "max_ms": 112.142, "p95_ms": 101.169}, "yolo11_640_fire_11class": {"avg_ms": 65.179, "count": 208, "max_ms": 89.611, "p95_ms": 83.984}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.245, "count": 208, "max_ms": 108.336, "p95_ms": 101.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.672, "count": 208, "max_ms": 177.574, "p95_ms": 160.24}, "read_interval": {"avg_ms": 144.367, "count": 207, "max_ms": 189.27, "p95_ms": 169.27}, "triton_wait": {"avg_ms": 80.732, "count": 208, "max_ms": 112.156, "p95_ms": 102.472}, "visualize": {"avg_ms": 5.158, "count": 208, "max_ms": 9.335, "p95_ms": 6.028}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 14}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5472.129, "limit_mb": 8796093022207.996}, "detail": {"pid": 6896, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:46:52+09:00", "last_error_summary": "[h264 @ 0x55de8668e540] error while decoding MB 76 22, bytestream -7", "last_exit_at": "2026-07-06T12:46:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:46:55+09:00", "pid": 6896, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.508, "name": "ffmpeg", "pid": 6896, "rss_mb": 146.508, "state": "S (sleeping)", "threads": 106}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 195.16, "sample_count": 1736, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 7.677}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1736, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:46:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.84, "last_frame_pushed_at": "2026-07-06T12:46:52+09:00", "last_frame_read_age_sec": 4.955, "last_frame_read_at": "2026-07-06T12:46:52+09:00", "mjpeg": {"age_sec": 4.817, "exists": true, "mtime": "2026-07-06T12:46:52.379292+09:00", "mtime_epoch": 1783309612.3792918, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169793}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T12:46:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1441.66, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.845, "pushed": 5.845, "read": 5.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.016, "count": 175, "max_ms": 9.262, "p95_ms": 8.501}, "gstreamer_push": {"avg_ms": 1.225, "count": 175, "max_ms": 4.101, "p95_ms": 1.973}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.848, "count": 175, "max_ms": 87.374, "p95_ms": 80.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.295, "count": 175, "max_ms": 112.142, "p95_ms": 101.59}, "yolo11_640_fire_11class": {"avg_ms": 65.26, "count": 175, "max_ms": 89.611, "p95_ms": 84.483}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.54, "count": 175, "max_ms": 108.336, "p95_ms": 101.747}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.843, "count": 175, "max_ms": 177.574, "p95_ms": 160.24}, "read_interval": {"avg_ms": 144.453, "count": 174, "max_ms": 189.27, "p95_ms": 169.27}, "triton_wait": {"avg_ms": 80.97, "count": 175, "max_ms": 112.156, "p95_ms": 103.521}, "visualize": {"avg_ms": 5.202, "count": 175, "max_ms": 9.335, "p95_ms": 6.039}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5310.234, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a46e7cb6c0] error while decoding MB 104 40, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:51:44+09:00", "last_error_summary": "[h264 @ 0x55a46e7cb6c0] error while decoding MB 104 40, bytestream -5", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 581.096, "last_started_at": "2026-07-06T12:42:03+09:00", "pid": 6527, "process": {"exists": false, "pid": 6527}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.988, "sample_count": 2200, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4561, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:51:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T12:51:44+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T12:51:44+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T12:51:44.403137+09:00", "mtime_epoch": 1783309904.403137, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227536}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:51:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.113, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.172, "pushed": 7.172, "read": 7.172}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.566, "count": 215, "max_ms": 8.591, "p95_ms": 6.445}, "gstreamer_push": {"avg_ms": 1.235, "count": 215, "max_ms": 5.098, "p95_ms": 1.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.38, "count": 215, "max_ms": 81.759, "p95_ms": 74.337}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.168, "count": 215, "max_ms": 113.665, "p95_ms": 102.781}, "yolo11_640_fire_11class": {"avg_ms": 61.909, "count": 215, "max_ms": 89.855, "p95_ms": 81.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.318, "count": 215, "max_ms": 101.319, "p95_ms": 94.308}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.156, "count": 215, "max_ms": 164.858, "p95_ms": 155.871}, "read_interval": {"avg_ms": 139.489, "count": 215, "max_ms": 179.558, "p95_ms": 166.022}, "triton_wait": {"avg_ms": 77.36, "count": 215, "max_ms": 113.675, "p95_ms": 102.794}, "visualize": {"avg_ms": 5.14, "count": 215, "max_ms": 7.891, "p95_ms": 5.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5309.539, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T12:51:44+09:00", "last_error_summary": "[h264 @ 0x55a46e7cb6c0] error while decoding MB 104 40, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:51:44+09:00", "last_error_summary": "[h264 @ 0x55a46e7cb6c0] error while decoding MB 104 40, bytestream -5", "last_exit_at": "2026-07-06T12:51:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 581.157, "last_started_at": "2026-07-06T12:42:03+09:00", "pid": 6527, "process": {"exists": false, "pid": 6527}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.988, "sample_count": 2201, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4562, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:51:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T12:51:44+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T12:51:44+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T12:51:44.403137+09:00", "mtime_epoch": 1783309904.403137, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227536}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:51:44+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.117, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.195, "pushed": 7.195, "read": 7.161}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.612, "count": 215, "max_ms": 13.078, "p95_ms": 6.475}, "gstreamer_push": {"avg_ms": 1.234, "count": 215, "max_ms": 5.098, "p95_ms": 1.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.419, "count": 215, "max_ms": 81.759, "p95_ms": 74.337}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.146, "count": 215, "max_ms": 113.665, "p95_ms": 102.781}, "yolo11_640_fire_11class": {"avg_ms": 61.938, "count": 215, "max_ms": 89.855, "p95_ms": 81.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.349, "count": 215, "max_ms": 101.319, "p95_ms": 94.308}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.355, "count": 215, "max_ms": 164.858, "p95_ms": 155.871}, "read_interval": {"avg_ms": 139.732, "count": 214, "max_ms": 179.558, "p95_ms": 166.022}, "triton_wait": {"avg_ms": 77.338, "count": 215, "max_ms": 113.675, "p95_ms": 102.794}, "visualize": {"avg_ms": 5.147, "count": 215, "max_ms": 7.891, "p95_ms": 5.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5309.531, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:51:44+09:00", "last_error_summary": "[h264 @ 0x55a46e7cb6c0] error while decoding MB 104 40, bytestream -5", "last_exit_at": "2026-07-06T12:51:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 581.163, "last_started_at": "2026-07-06T12:42:03+09:00", "pid": 6527, "process": {"exists": false, "pid": 6527}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.988, "sample_count": 2201, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4562, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:51:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T12:51:44+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T12:51:44+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T12:51:44.403137+09:00", "mtime_epoch": 1783309904.403137, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227536}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:51:44+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.117, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.193, "pushed": 7.193, "read": 7.16}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.612, "count": 215, "max_ms": 13.078, "p95_ms": 6.475}, "gstreamer_push": {"avg_ms": 1.234, "count": 215, "max_ms": 5.098, "p95_ms": 1.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.419, "count": 215, "max_ms": 81.759, "p95_ms": 74.337}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.146, "count": 215, "max_ms": 113.665, "p95_ms": 102.781}, "yolo11_640_fire_11class": {"avg_ms": 61.938, "count": 215, "max_ms": 89.855, "p95_ms": 81.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.349, "count": 215, "max_ms": 101.319, "p95_ms": 94.308}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.355, "count": 215, "max_ms": 164.858, "p95_ms": 155.871}, "read_interval": {"avg_ms": 139.732, "count": 214, "max_ms": 179.558, "p95_ms": 166.022}, "triton_wait": {"avg_ms": 77.338, "count": 215, "max_ms": 113.675, "p95_ms": 102.794}, "visualize": {"avg_ms": 5.147, "count": 215, "max_ms": 7.891, "p95_ms": 5.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5479.988, "limit_mb": 8796093022207.996}, "detail": {"pid": 7003, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T12:51:44+09:00", "last_error_summary": "[h264 @ 0x55a46e7cb6c0] error while decoding MB 104 40, bytestream -5", "last_exit_at": "2026-07-06T12:51:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T12:51:46+09:00", "pid": 7003, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.848, "name": "ffmpeg", "pid": 7003, "rss_mb": 194.848, "state": "S (sleeping)", "threads": 106}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.004, "sample_count": 2168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4562, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T12:51:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.477, "last_frame_pushed_at": "2026-07-06T12:51:44+09:00", "last_frame_read_age_sec": 4.597, "last_frame_read_at": "2026-07-06T12:51:44+09:00", "mjpeg": {"age_sec": 4.461, "exists": true, "mtime": "2026-07-06T12:51:44.527140+09:00", "mtime_epoch": 1783309904.5271401, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222661}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T12:51:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.108, "pushed": 6.108, "read": 6.075}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.614, "count": 182, "max_ms": 13.078, "p95_ms": 6.505}, "gstreamer_push": {"avg_ms": 1.201, "count": 183, "max_ms": 5.098, "p95_ms": 1.37}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.655, "count": 182, "max_ms": 81.759, "p95_ms": 74.953}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.881, "count": 182, "max_ms": 113.665, "p95_ms": 102.781}, "yolo11_640_fire_11class": {"avg_ms": 62.397, "count": 182, "max_ms": 89.855, "p95_ms": 81.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.005, "count": 182, "max_ms": 101.319, "p95_ms": 94.647}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.914, "count": 183, "max_ms": 164.858, "p95_ms": 155.927}, "read_interval": {"avg_ms": 140.256, "count": 182, "max_ms": 179.558, "p95_ms": 166.077}, "triton_wait": {"avg_ms": 78.028, "count": 182, "max_ms": 113.675, "p95_ms": 102.794}, "visualize": {"avg_ms": 5.154, "count": 182, "max_ms": 7.891, "p95_ms": 5.855}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 82}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5371.367, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55702d776ac0] cabac decode of qscale diff failed at 12 5 [h264 @ 0x55702d776ac0] error while decoding MB 12 5, bytestream 7167 [h264 @ 0x55702caf1540] top block unavailable for requested intra mode [h264 @ 0x55702caf1540] error while decoding MB 16 0, bytestream 2186 [h264 @ 0x55702d36c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55702d36c240] error while decoding MB 0 40, bytestream 2469 [h264 @ 0x55702d6a7c40] cabac decode of qscale diff failed at 27 27 [h264 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:01:25+09:00", "last_error_summary": "[h264 @ 0x55702d776ac0] cabac decode of qscale diff failed at 12 5 [h264 @ 0x55702d776ac0] error while decoding MB 12 5, bytestream 7167 [h264 @ 0x55702caf1540] top block unavailable for requested intra mode [h264 @ 0x55702caf1540] error while decoding MB 16 0, bytestream 2186 [h264 @ 0x55702d36c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55702d36c240] error while decoding MB 0 40, bytestream 2469 [h264 @ 0x55702d6a7c40] cabac decode of qscale diff failed at 27 27 [h264 ", "last_exit_at": "2026-07-06T12:46:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 870.733, "last_started_at": "2026-07-06T12:46:55+09:00", "pid": 6896, "process": {"exists": false, "pid": 6896}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.373, "sample_count": 2535, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9849, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:01:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T13:01:25+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-06T13:01:25+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-06T13:01:25.864318+09:00", "mtime_epoch": 1783310485.864318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192263}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:01:25+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1443.297, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 148, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.946, "pushed": 4.946, "read": 4.946}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.425, "count": 148, "max_ms": 5.677, "p95_ms": 4.758}, "gstreamer_push": {"avg_ms": 1.662, "count": 148, "max_ms": 7.17, "p95_ms": 3.956}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.716, "count": 148, "max_ms": 85.584, "p95_ms": 76.681}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.701, "count": 148, "max_ms": 114.454, "p95_ms": 97.608}, "yolo11_640_fire_11class": {"avg_ms": 62.898, "count": 148, "max_ms": 87.597, "p95_ms": 82.625}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.377, "count": 148, "max_ms": 100.046, "p95_ms": 94.299}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.631, "count": 148, "max_ms": 126.538, "p95_ms": 110.288}, "read_interval": {"avg_ms": 202.282, "count": 148, "max_ms": 9286.813, "p95_ms": 126.45}, "triton_wait": {"avg_ms": 74.307, "count": 148, "max_ms": 114.467, "p95_ms": 98.273}, "visualize": {"avg_ms": 3.739, "count": 148, "max_ms": 5.739, "p95_ms": 4.385}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 82}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5371.207, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:01:25+09:00", "last_error_summary": "[h264 @ 0x55702d776ac0] cabac decode of qscale diff failed at 12 5 [h264 @ 0x55702d776ac0] error while decoding MB 12 5, bytestream 7167 [h264 @ 0x55702caf1540] top block unavailable for requested intra mode [h264 @ 0x55702caf1540] error while decoding MB 16 0, bytestream 2186 [h264 @ 0x55702d36c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55702d36c240] error while decoding MB 0 40, bytestream 2469 [h264 @ 0x55702d6a7c40] cabac decode of qscale diff failed at 27 27 [h264 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:01:25+09:00", "last_error_summary": "[h264 @ 0x55702d776ac0] cabac decode of qscale diff failed at 12 5 [h264 @ 0x55702d776ac0] error while decoding MB 12 5, bytestream 7167 [h264 @ 0x55702caf1540] top block unavailable for requested intra mode [h264 @ 0x55702caf1540] error while decoding MB 16 0, bytestream 2186 [h264 @ 0x55702d36c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55702d36c240] error while decoding MB 0 40, bytestream 2469 [h264 @ 0x55702d6a7c40] cabac decode of qscale diff failed at 27 27 [h264 ", "last_exit_at": "2026-07-06T13:01:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 870.76, "last_started_at": "2026-07-06T12:46:55+09:00", "pid": 6896, "process": {"exists": false, "pid": 6896}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.373, "sample_count": 2536, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9850, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:01:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T13:01:25+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-06T13:01:25+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-06T13:01:25.864318+09:00", "mtime_epoch": 1783310485.864318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192263}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:01:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1443.297, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.975, "pushed": 4.975, "read": 4.941}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.458, "count": 149, "max_ms": 8.336, "p95_ms": 4.839}, "gstreamer_push": {"avg_ms": 1.66, "count": 149, "max_ms": 7.17, "p95_ms": 3.956}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.679, "count": 149, "max_ms": 85.584, "p95_ms": 76.681}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.651, "count": 149, "max_ms": 114.454, "p95_ms": 97.608}, "yolo11_640_fire_11class": {"avg_ms": 62.875, "count": 149, "max_ms": 87.597, "p95_ms": 82.625}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.328, "count": 149, "max_ms": 100.046, "p95_ms": 94.299}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.622, "count": 149, "max_ms": 126.538, "p95_ms": 110.288}, "read_interval": {"avg_ms": 202.282, "count": 148, "max_ms": 9286.813, "p95_ms": 126.45}, "triton_wait": {"avg_ms": 74.253, "count": 149, "max_ms": 114.467, "p95_ms": 98.273}, "visualize": {"avg_ms": 3.751, "count": 149, "max_ms": 5.739, "p95_ms": 4.415}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 82}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5371.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:01:25+09:00", "last_error_summary": "[h264 @ 0x55702d776ac0] cabac decode of qscale diff failed at 12 5 [h264 @ 0x55702d776ac0] error while decoding MB 12 5, bytestream 7167 [h264 @ 0x55702caf1540] top block unavailable for requested intra mode [h264 @ 0x55702caf1540] error while decoding MB 16 0, bytestream 2186 [h264 @ 0x55702d36c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55702d36c240] error while decoding MB 0 40, bytestream 2469 [h264 @ 0x55702d6a7c40] cabac decode of qscale diff failed at 27 27 [h264 ", "last_exit_at": "2026-07-06T13:01:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 870.768, "last_started_at": "2026-07-06T12:46:55+09:00", "pid": 6896, "process": {"exists": false, "pid": 6896}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.373, "sample_count": 2536, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9850, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:01:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T13:01:25+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-06T13:01:25+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-06T13:01:25.864318+09:00", "mtime_epoch": 1783310485.864318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192263}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:01:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1443.297, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.974, "pushed": 4.974, "read": 4.94}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.458, "count": 149, "max_ms": 8.336, "p95_ms": 4.839}, "gstreamer_push": {"avg_ms": 1.66, "count": 149, "max_ms": 7.17, "p95_ms": 3.956}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.679, "count": 149, "max_ms": 85.584, "p95_ms": 76.681}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.651, "count": 149, "max_ms": 114.454, "p95_ms": 97.608}, "yolo11_640_fire_11class": {"avg_ms": 62.875, "count": 149, "max_ms": 87.597, "p95_ms": 82.625}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.328, "count": 149, "max_ms": 100.046, "p95_ms": 94.299}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.622, "count": 149, "max_ms": 126.538, "p95_ms": 110.288}, "read_interval": {"avg_ms": 202.282, "count": 148, "max_ms": 9286.813, "p95_ms": 126.45}, "triton_wait": {"avg_ms": 74.253, "count": 149, "max_ms": 114.467, "p95_ms": 98.273}, "visualize": {"avg_ms": 3.751, "count": 149, "max_ms": 5.739, "p95_ms": 4.415}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 82}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5481.047, "limit_mb": 8796093022207.996}, "detail": {"pid": 7155, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:01:25+09:00", "last_error_summary": "[h264 @ 0x55702d776ac0] cabac decode of qscale diff failed at 12 5 [h264 @ 0x55702d776ac0] error while decoding MB 12 5, bytestream 7167 [h264 @ 0x55702caf1540] top block unavailable for requested intra mode [h264 @ 0x55702caf1540] error while decoding MB 16 0, bytestream 2186 [h264 @ 0x55702d36c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55702d36c240] error while decoding MB 0 40, bytestream 2469 [h264 @ 0x55702d6a7c40] cabac decode of qscale diff failed at 27 27 [h264 ", "last_exit_at": "2026-07-06T13:01:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:01:28+09:00", "pid": 7155, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.297, "name": "ffmpeg", "pid": 7155, "rss_mb": 139.297, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.373, "sample_count": 2491, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9850, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:01:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.454, "last_frame_pushed_at": "2026-07-06T13:01:25+09:00", "last_frame_read_age_sec": 4.541, "last_frame_read_at": "2026-07-06T13:01:25+09:00", "mjpeg": {"age_sec": 4.441, "exists": true, "mtime": "2026-07-06T13:01:25.960320+09:00", "mtime_epoch": 1783310485.96032, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192301}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:01:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1443.297, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.638, "pushed": 3.638, "read": 3.605}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.477, "count": 109, "max_ms": 8.336, "p95_ms": 4.758}, "gstreamer_push": {"avg_ms": 1.587, "count": 109, "max_ms": 4.89, "p95_ms": 3.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.772, "count": 109, "max_ms": 80.865, "p95_ms": 75.0}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.503, "count": 109, "max_ms": 104.435, "p95_ms": 95.107}, "yolo11_640_fire_11class": {"avg_ms": 61.031, "count": 109, "max_ms": 83.849, "p95_ms": 80.842}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.561, "count": 109, "max_ms": 100.046, "p95_ms": 94.267}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.571, "count": 109, "max_ms": 116.201, "p95_ms": 108.956}, "read_interval": {"avg_ms": 236.261, "count": 108, "max_ms": 9286.813, "p95_ms": 123.571}, "triton_wait": {"avg_ms": 72.17, "count": 109, "max_ms": 104.447, "p95_ms": 96.907}, "visualize": {"avg_ms": 3.789, "count": 109, "max_ms": 5.739, "p95_ms": 4.471}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 97}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5312.652, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562e6a5518c0] error while decoding MB 4 29, bytestream -21", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:06:30+09:00", "last_error_summary": "[h264 @ 0x562e6a5518c0] error while decoding MB 4 29, bytestream -21", "last_exit_at": "2026-07-06T12:51:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 883.681, "last_started_at": "2026-07-06T12:51:46+09:00", "pid": 7003, "process": {"exists": false, "pid": 7003}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.015, "sample_count": 2249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11650, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:06:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T13:06:30+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T13:06:30+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T13:06:30.614620+09:00", "mtime_epoch": 1783310790.6146204, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 219629}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:06:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1314.441, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 250, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.336, "pushed": 8.336, "read": 8.336}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.15, "count": 249, "max_ms": 8.504, "p95_ms": 6.225}, "gstreamer_push": {"avg_ms": 1.447, "count": 250, "max_ms": 6.748, "p95_ms": 3.419}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.396, "count": 249, "max_ms": 93.355, "p95_ms": 77.639}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.528, "count": 249, "max_ms": 124.671, "p95_ms": 106.275}, "yolo11_640_fire_11class": {"avg_ms": 64.867, "count": 249, "max_ms": 94.377, "p95_ms": 83.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.725, "count": 249, "max_ms": 106.783, "p95_ms": 97.438}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.171, "count": 250, "max_ms": 174.944, "p95_ms": 151.117}, "read_interval": {"avg_ms": 120.108, "count": 250, "max_ms": 187.717, "p95_ms": 160.384}, "triton_wait": {"avg_ms": 80.834, "count": 249, "max_ms": 124.688, "p95_ms": 106.285}, "visualize": {"avg_ms": 4.285, "count": 249, "max_ms": 7.445, "p95_ms": 5.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 97}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5312.414, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:06:30+09:00", "last_error_summary": "[h264 @ 0x562e6a5518c0] error while decoding MB 4 29, bytestream -21", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:06:30+09:00", "last_error_summary": "[h264 @ 0x562e6a5518c0] error while decoding MB 4 29, bytestream -21", "last_exit_at": "2026-07-06T13:06:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 883.713, "last_started_at": "2026-07-06T12:51:46+09:00", "pid": 7003, "process": {"exists": false, "pid": 7003}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.015, "sample_count": 2249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11651, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:06:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T13:06:30+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-06T13:06:30+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-06T13:06:30.614620+09:00", "mtime_epoch": 1783310790.6146204, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 219629}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:06:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1314.441, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.363, "pushed": 8.363, "read": 8.33}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.15, "count": 250, "max_ms": 8.504, "p95_ms": 6.225}, "gstreamer_push": {"avg_ms": 1.448, "count": 250, "max_ms": 6.748, "p95_ms": 3.419}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.37, "count": 250, "max_ms": 93.355, "p95_ms": 77.639}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.487, "count": 250, "max_ms": 124.671, "p95_ms": 106.275}, "yolo11_640_fire_11class": {"avg_ms": 64.879, "count": 250, "max_ms": 94.377, "p95_ms": 83.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.719, "count": 250, "max_ms": 106.783, "p95_ms": 97.438}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.108, "count": 250, "max_ms": 174.944, "p95_ms": 151.117}, "read_interval": {"avg_ms": 120.145, "count": 249, "max_ms": 187.717, "p95_ms": 160.384}, "triton_wait": {"avg_ms": 80.796, "count": 250, "max_ms": 124.688, "p95_ms": 106.285}, "visualize": {"avg_ms": 4.289, "count": 250, "max_ms": 7.445, "p95_ms": 5.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 97}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5312.23, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:06:30+09:00", "last_error_summary": "[h264 @ 0x562e6a5518c0] error while decoding MB 4 29, bytestream -21", "last_exit_at": "2026-07-06T13:06:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 883.722, "last_started_at": "2026-07-06T12:51:46+09:00", "pid": 7003, "process": {"exists": false, "pid": 7003}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.015, "sample_count": 2249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11651, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:06:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T13:06:30+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-06T13:06:30+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-06T13:06:30.614620+09:00", "mtime_epoch": 1783310790.6146204, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 219629}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:06:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1314.441, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.361, "pushed": 8.361, "read": 8.327}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.15, "count": 250, "max_ms": 8.504, "p95_ms": 6.225}, "gstreamer_push": {"avg_ms": 1.448, "count": 250, "max_ms": 6.748, "p95_ms": 3.419}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.37, "count": 250, "max_ms": 93.355, "p95_ms": 77.639}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.487, "count": 250, "max_ms": 124.671, "p95_ms": 106.275}, "yolo11_640_fire_11class": {"avg_ms": 64.879, "count": 250, "max_ms": 94.377, "p95_ms": 83.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.719, "count": 250, "max_ms": 106.783, "p95_ms": 97.438}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.108, "count": 250, "max_ms": 174.944, "p95_ms": 151.117}, "read_interval": {"avg_ms": 120.145, "count": 249, "max_ms": 187.717, "p95_ms": 160.384}, "triton_wait": {"avg_ms": 80.796, "count": 250, "max_ms": 124.688, "p95_ms": 106.285}, "visualize": {"avg_ms": 4.289, "count": 250, "max_ms": 7.445, "p95_ms": 5.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 97}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5332.32, "limit_mb": 8796093022207.996}, "detail": {"pid": 7262, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:06:30+09:00", "last_error_summary": "[h264 @ 0x562e6a5518c0] error while decoding MB 4 29, bytestream -21", "last_exit_at": "2026-07-06T13:06:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:06:33+09:00", "pid": 7262, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.676, "name": "ffmpeg", "pid": 7262, "rss_mb": 57.676, "state": "S (sleeping)", "threads": 1}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.015, "sample_count": 2214, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11651, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:06:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.833, "last_frame_pushed_at": "2026-07-06T13:06:30+09:00", "last_frame_read_age_sec": 4.922, "last_frame_read_at": "2026-07-06T13:06:30+09:00", "mjpeg": {"age_sec": 4.816, "exists": true, "mtime": "2026-07-06T13:06:30.710622+09:00", "mtime_epoch": 1783310790.7106223, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 219629}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:06:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1314.441, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.186, "pushed": 7.186, "read": 7.153}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.896, "count": 215, "max_ms": 7.581, "p95_ms": 5.916}, "gstreamer_push": {"avg_ms": 1.451, "count": 215, "max_ms": 6.748, "p95_ms": 3.279}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.469, "count": 215, "max_ms": 93.355, "p95_ms": 78.378}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.597, "count": 215, "max_ms": 124.671, "p95_ms": 106.809}, "yolo11_640_fire_11class": {"avg_ms": 65.785, "count": 215, "max_ms": 94.377, "p95_ms": 84.434}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.554, "count": 215, "max_ms": 106.783, "p95_ms": 97.514}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.998, "count": 215, "max_ms": 174.944, "p95_ms": 145.852}, "read_interval": {"avg_ms": 117.338, "count": 214, "max_ms": 187.717, "p95_ms": 156.576}, "triton_wait": {"avg_ms": 81.92, "count": 215, "max_ms": 124.688, "p95_ms": 106.822}, "visualize": {"avg_ms": 4.136, "count": 215, "max_ms": 7.445, "p95_ms": 5.342}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5322.371, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f478b5b000] error while decoding MB 97 30, bytestream -29 [in#0/rtsp @ 0x55f478b13e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f478b13e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:08:05+09:00", "last_error_summary": "[h264 @ 0x55f478b5b000] error while decoding MB 97 30, bytestream -29 [in#0/rtsp @ 0x55f478b13e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f478b13e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:06:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 91.701, "last_started_at": "2026-07-06T13:06:33+09:00", "pid": 7262, "process": {"exists": false, "pid": 7262}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.455, "sample_count": 2023, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12119, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:08:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T13:08:05+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T13:08:05+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T13:08:05.168534+09:00", "mtime_epoch": 1783310885.1685336, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222424}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:08:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.371, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.675, "pushed": 0.675, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.408, "count": 12, "max_ms": 6.636, "p95_ms": 6.636}, "gstreamer_push": {"avg_ms": 1.341, "count": 12, "max_ms": 1.46, "p95_ms": 1.46}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.766, "count": 12, "max_ms": 69.295, "p95_ms": 69.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.396, "count": 12, "max_ms": 104.653, "p95_ms": 104.653}, "yolo11_640_fire_11class": {"avg_ms": 61.872, "count": 12, "max_ms": 79.792, "p95_ms": 79.792}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.787, "count": 12, "max_ms": 97.008, "p95_ms": 97.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.79, "count": 12, "max_ms": 138.187, "p95_ms": 138.187}, "read_interval": {"avg_ms": 2508.008, "count": 13, "max_ms": 16586.339, "p95_ms": 16586.339}, "triton_wait": {"avg_ms": 76.407, "count": 12, "max_ms": 104.665, "p95_ms": 104.665}, "visualize": {"avg_ms": 3.849, "count": 12, "max_ms": 5.489, "p95_ms": 5.489}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.778, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5321.676, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:08:05+09:00", "last_error_summary": "[h264 @ 0x55f478b5b000] error while decoding MB 97 30, bytestream -29 [in#0/rtsp @ 0x55f478b13e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f478b13e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:08:05+09:00", "last_error_summary": "[h264 @ 0x55f478b5b000] error while decoding MB 97 30, bytestream -29 [in#0/rtsp @ 0x55f478b13e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f478b13e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:08:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 91.747, "last_started_at": "2026-07-06T13:06:33+09:00", "pid": 7262, "process": {"exists": false, "pid": 7262}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.455, "sample_count": 2023, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12120, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:08:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T13:08:05+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-06T13:08:05+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-06T13:08:05.168534+09:00", "mtime_epoch": 1783310885.1685336, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222424}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:08:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.371, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.729, "pushed": 0.729, "read": 0.729}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.606, "count": 13, "max_ms": 6.636, "p95_ms": 6.636}, "gstreamer_push": {"avg_ms": 1.419, "count": 13, "max_ms": 2.35, "p95_ms": 2.35}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.139, "count": 13, "max_ms": 69.295, "p95_ms": 69.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.563, "count": 13, "max_ms": 104.653, "p95_ms": 104.653}, "yolo11_640_fire_11class": {"avg_ms": 61.915, "count": 13, "max_ms": 79.792, "p95_ms": 79.792}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.974, "count": 13, "max_ms": 97.008, "p95_ms": 97.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.377, "count": 13, "max_ms": 138.187, "p95_ms": 138.187}, "read_interval": {"avg_ms": 2508.008, "count": 13, "max_ms": 16586.339, "p95_ms": 16586.339}, "triton_wait": {"avg_ms": 76.575, "count": 13, "max_ms": 104.665, "p95_ms": 104.665}, "visualize": {"avg_ms": 4.008, "count": 13, "max_ms": 5.927, "p95_ms": 5.927}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.823, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5321.695, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:08:05+09:00", "last_error_summary": "[h264 @ 0x55f478b5b000] error while decoding MB 97 30, bytestream -29 [in#0/rtsp @ 0x55f478b13e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f478b13e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:08:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 91.756, "last_started_at": "2026-07-06T13:06:33+09:00", "pid": 7262, "process": {"exists": false, "pid": 7262}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.455, "sample_count": 2023, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12120, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:08:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T13:08:05+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T13:08:05+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T13:08:05.168534+09:00", "mtime_epoch": 1783310885.1685336, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222424}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:08:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.371, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.729, "pushed": 0.729, "read": 0.729}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.606, "count": 13, "max_ms": 6.636, "p95_ms": 6.636}, "gstreamer_push": {"avg_ms": 1.419, "count": 13, "max_ms": 2.35, "p95_ms": 2.35}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.139, "count": 13, "max_ms": 69.295, "p95_ms": 69.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.563, "count": 13, "max_ms": 104.653, "p95_ms": 104.653}, "yolo11_640_fire_11class": {"avg_ms": 61.915, "count": 13, "max_ms": 79.792, "p95_ms": 79.792}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.974, "count": 13, "max_ms": 97.008, "p95_ms": 97.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.377, "count": 13, "max_ms": 138.187, "p95_ms": 138.187}, "read_interval": {"avg_ms": 2508.008, "count": 13, "max_ms": 16586.339, "p95_ms": 16586.339}, "triton_wait": {"avg_ms": 76.575, "count": 13, "max_ms": 104.665, "p95_ms": 104.665}, "visualize": {"avg_ms": 4.008, "count": 13, "max_ms": 5.927, "p95_ms": 5.927}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.832, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5486.391, "limit_mb": 8796093022207.996}, "detail": {"pid": 7414, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:08:05+09:00", "last_error_summary": "[h264 @ 0x55f478b5b000] error while decoding MB 97 30, bytestream -29 [in#0/rtsp @ 0x55f478b13e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f478b13e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:08:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:08:08+09:00", "pid": 7414, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.691, "name": "ffmpeg", "pid": 7414, "rss_mb": 194.691, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.45, "sample_count": 1986, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12120, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:08:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.873, "last_frame_pushed_at": "2026-07-06T13:08:05+09:00", "last_frame_read_age_sec": 4.975, "last_frame_read_at": "2026-07-06T13:08:05+09:00", "mjpeg": {"age_sec": 4.854, "exists": true, "mtime": "2026-07-06T13:08:05.284536+09:00", "mtime_epoch": 1783310885.284536, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221241}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:08:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.371, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.573, "pushed": 0.573, "read": 0.573}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.606, "count": 13, "max_ms": 6.636, "p95_ms": 6.636}, "gstreamer_push": {"avg_ms": 1.419, "count": 13, "max_ms": 2.35, "p95_ms": 2.35}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.139, "count": 13, "max_ms": 69.295, "p95_ms": 69.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.563, "count": 13, "max_ms": 104.653, "p95_ms": 104.653}, "yolo11_640_fire_11class": {"avg_ms": 61.915, "count": 13, "max_ms": 79.792, "p95_ms": 79.792}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.974, "count": 13, "max_ms": 97.008, "p95_ms": 97.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.377, "count": 13, "max_ms": 138.187, "p95_ms": 138.187}, "read_interval": {"avg_ms": 2508.008, "count": 13, "max_ms": 16586.339, "p95_ms": 16586.339}, "triton_wait": {"avg_ms": 76.575, "count": 13, "max_ms": 104.665, "p95_ms": 104.665}, "visualize": {"avg_ms": 4.008, "count": 13, "max_ms": 5.927, "p95_ms": 5.927}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.689, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 106}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5627.293, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:08:05+09:00", "last_error_summary": "[h264 @ 0x55f478b5b000] error while decoding MB 97 30, bytestream -29 [in#0/rtsp @ 0x55f478b13e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f478b13e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:08:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.843, "last_started_at": "2026-07-06T13:08:08+09:00", "pid": 7414, "process": {"exists": true, "fd_count": 5, "hwm_mb": 332.594, "name": "ffmpeg", "pid": 7414, "rss_mb": 332.594, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.652, "sample_count": 1835, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:09:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.057, "last_frame_pushed_at": "2026-07-06T13:09:27+09:00", "last_frame_read_age_sec": 30.161, "last_frame_read_at": "2026-07-06T13:09:27+09:00", "mjpeg": {"age_sec": 30.038, "exists": true, "mtime": "2026-07-06T13:09:27.942196+09:00", "mtime_epoch": 1783310967.9421964, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232030}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:09:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 106}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5617.305, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:08:05+09:00", "last_error_summary": "[h264 @ 0x55f478b5b000] error while decoding MB 97 30, bytestream -29 [in#0/rtsp @ 0x55f478b13e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f478b13e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:08:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.855, "last_started_at": "2026-07-06T13:08:08+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.652, "sample_count": 1835, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:09:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.069, "last_frame_pushed_at": "2026-07-06T13:09:27+09:00", "last_frame_read_age_sec": 30.173, "last_frame_read_at": "2026-07-06T13:09:27+09:00", "mjpeg": {"age_sec": 30.05, "exists": true, "mtime": "2026-07-06T13:09:27.942196+09:00", "mtime_epoch": 1783310967.9421964, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232030}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:09:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 106}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5324.453, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fdda4a3e40] error while decoding MB 91 33, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:09:58+09:00", "last_error_summary": "[h264 @ 0x55fdda4a3e40] error while decoding MB 91 33, bytestream -11", "last_exit_at": "2026-07-06T13:08:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.907, "last_started_at": "2026-07-06T13:08:08+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.652, "sample_count": 1835, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:09:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.121, "last_frame_pushed_at": "2026-07-06T13:09:27+09:00", "last_frame_read_age_sec": 30.224, "last_frame_read_at": "2026-07-06T13:09:27+09:00", "mjpeg": {"age_sec": 30.102, "exists": true, "mtime": "2026-07-06T13:09:27.942196+09:00", "mtime_epoch": 1783310967.9421964, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232030}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:09:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 106}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5346.332, "limit_mb": 8796093022207.996}, "detail": {"pid": 7566, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:09:58+09:00", "last_error_summary": "[h264 @ 0x55fdda4a3e40] error while decoding MB 91 33, bytestream -11", "last_exit_at": "2026-07-06T13:08:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:10:00+09:00", "pid": 7566, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.457, "name": "ffmpeg", "pid": 7566, "rss_mb": 57.457, "state": "S (sleeping)", "threads": 1}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.316, "sample_count": 1805, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:09:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.363, "last_frame_pushed_at": "2026-07-06T13:09:27+09:00", "last_frame_read_age_sec": 34.467, "last_frame_read_at": "2026-07-06T13:09:27+09:00", "mjpeg": {"age_sec": 34.344, "exists": true, "mtime": "2026-07-06T13:09:27.942196+09:00", "mtime_epoch": 1783310967.9421964, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232030}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:10:02+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1322.805, "name": "celery", "pid": 267, "rss_mb": 1312.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 118}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5332.629, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560d971fdb00] error while decoding MB 109 57, bytestream -7 [in#0/rtsp @ 0x560d96b5ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560d96b5ee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:13:21+09:00", "last_error_summary": "[h264 @ 0x560d971fdb00] error while decoding MB 109 57, bytestream -7 [in#0/rtsp @ 0x560d96b5ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560d96b5ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:08:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 201.304, "last_started_at": "2026-07-06T13:10:00+09:00", "pid": 7566, "process": {"exists": false, "pid": 7566}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.27, "sample_count": 2057, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14271, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:13:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T13:13:21+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-06T13:13:21+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T13:13:21.530843+09:00", "mtime_epoch": 1783311201.5308425, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224055}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:13:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1314.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.401, "pushed": 1.401, "read": 1.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.957, "count": 42, "max_ms": 3.091, "p95_ms": 3.041}, "gstreamer_push": {"avg_ms": 1.431, "count": 42, "max_ms": 3.913, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.434, "count": 42, "max_ms": 77.637, "p95_ms": 60.571}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.918, "count": 42, "max_ms": 92.787, "p95_ms": 91.494}, "yolo11_640_fire_11class": {"avg_ms": 56.446, "count": 42, "max_ms": 78.84, "p95_ms": 68.538}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.952, "count": 42, "max_ms": 87.194, "p95_ms": 69.501}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.444, "count": 42, "max_ms": 104.363, "p95_ms": 102.92}, "read_interval": {"avg_ms": 714.57, "count": 42, "max_ms": 16709.558, "p95_ms": 119.226}, "triton_wait": {"avg_ms": 63.93, "count": 42, "max_ms": 92.801, "p95_ms": 91.507}, "visualize": {"avg_ms": 3.65, "count": 42, "max_ms": 3.974, "p95_ms": 3.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 118}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5332.559, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:13:21+09:00", "last_error_summary": "[h264 @ 0x560d971fdb00] error while decoding MB 109 57, bytestream -7 [in#0/rtsp @ 0x560d96b5ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560d96b5ee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:13:21+09:00", "last_error_summary": "[h264 @ 0x560d971fdb00] error while decoding MB 109 57, bytestream -7 [in#0/rtsp @ 0x560d96b5ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560d96b5ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:13:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 201.322, "last_started_at": "2026-07-06T13:10:00+09:00", "pid": 7566, "process": {"exists": false, "pid": 7566}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.27, "sample_count": 2057, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:13:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T13:13:21+09:00", "last_frame_read_age_sec": 0.087, "last_frame_read_at": "2026-07-06T13:13:21+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-06T13:13:21.530843+09:00", "mtime_epoch": 1783311201.5308425, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224055}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:13:21+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1314.285, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.434, "pushed": 1.434, "read": 1.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.956, "count": 43, "max_ms": 3.091, "p95_ms": 3.041}, "gstreamer_push": {"avg_ms": 1.428, "count": 43, "max_ms": 3.913, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.411, "count": 43, "max_ms": 77.637, "p95_ms": 60.571}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.852, "count": 43, "max_ms": 92.787, "p95_ms": 91.494}, "yolo11_640_fire_11class": {"avg_ms": 56.46, "count": 43, "max_ms": 78.84, "p95_ms": 68.538}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.041, "count": 43, "max_ms": 87.194, "p95_ms": 69.501}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.535, "count": 43, "max_ms": 104.363, "p95_ms": 102.92}, "read_interval": {"avg_ms": 714.57, "count": 42, "max_ms": 16709.558, "p95_ms": 119.226}, "triton_wait": {"avg_ms": 63.88, "count": 43, "max_ms": 92.801, "p95_ms": 91.507}, "visualize": {"avg_ms": 3.706, "count": 43, "max_ms": 6.037, "p95_ms": 3.816}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 118}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5332.277, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:13:21+09:00", "last_error_summary": "[h264 @ 0x560d971fdb00] error while decoding MB 109 57, bytestream -7 [in#0/rtsp @ 0x560d96b5ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560d96b5ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:13:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 201.33, "last_started_at": "2026-07-06T13:10:00+09:00", "pid": 7566, "process": {"exists": false, "pid": 7566}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.27, "sample_count": 2057, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:13:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T13:13:21+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-06T13:13:21+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-06T13:13:21.530843+09:00", "mtime_epoch": 1783311201.5308425, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224055}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:13:21+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1314.285, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.4, "pushed": 1.433, "read": 1.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.955, "count": 42, "max_ms": 3.091, "p95_ms": 3.041}, "gstreamer_push": {"avg_ms": 1.428, "count": 43, "max_ms": 3.913, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.145, "count": 42, "max_ms": 77.637, "p95_ms": 60.557}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.163, "count": 42, "max_ms": 92.18, "p95_ms": 90.42}, "yolo11_640_fire_11class": {"avg_ms": 56.172, "count": 42, "max_ms": 78.84, "p95_ms": 65.468}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.768, "count": 42, "max_ms": 87.194, "p95_ms": 69.32}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.849, "count": 42, "max_ms": 103.883, "p95_ms": 101.803}, "read_interval": {"avg_ms": 714.57, "count": 42, "max_ms": 16709.558, "p95_ms": 119.226}, "triton_wait": {"avg_ms": 63.192, "count": 42, "max_ms": 92.191, "p95_ms": 90.43}, "visualize": {"avg_ms": 3.706, "count": 42, "max_ms": 6.037, "p95_ms": 3.816}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 118}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5511.656, "limit_mb": 8796093022207.996}, "detail": {"pid": 7718, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:13:21+09:00", "last_error_summary": "[h264 @ 0x560d971fdb00] error while decoding MB 109 57, bytestream -7 [in#0/rtsp @ 0x560d96b5ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560d96b5ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:13:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:13:24+09:00", "pid": 7718, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.746, "name": "ffmpeg", "pid": 7718, "rss_mb": 206.746, "state": "S (sleeping)", "threads": 106}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.069, "sample_count": 2015, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:13:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.495, "last_frame_pushed_at": "2026-07-06T13:13:21+09:00", "last_frame_read_age_sec": 4.576, "last_frame_read_at": "2026-07-06T13:13:21+09:00", "mjpeg": {"age_sec": 4.479, "exists": true, "mtime": "2026-07-06T13:13:21.618844+09:00", "mtime_epoch": 1783311201.6188443, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222842}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:13:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1314.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.587, "pushed": 0.587, "read": 0.587}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.926, "count": 13, "max_ms": 2.959, "p95_ms": 2.959}, "gstreamer_push": {"avg_ms": 1.295, "count": 13, "max_ms": 1.36, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.166, "count": 13, "max_ms": 49.896, "p95_ms": 49.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.262, "count": 13, "max_ms": 82.704, "p95_ms": 82.704}, "yolo11_640_fire_11class": {"avg_ms": 53.93, "count": 13, "max_ms": 59.964, "p95_ms": 59.964}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.027, "count": 13, "max_ms": 61.769, "p95_ms": 61.769}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.109, "count": 13, "max_ms": 94.363, "p95_ms": 94.363}, "read_interval": {"avg_ms": 2105.059, "count": 13, "max_ms": 16709.558, "p95_ms": 16709.558}, "triton_wait": {"avg_ms": 59.328, "count": 13, "max_ms": 82.719, "p95_ms": 82.719}, "visualize": {"avg_ms": 3.786, "count": 13, "max_ms": 6.037, "p95_ms": 6.037}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.153, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5350.711, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555cf49ff000] error while decoding MB 83 50, bytestream -7 [in#0/rtsp @ 0x555cf4954e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555cf4954e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:17:08+09:00", "last_error_summary": "[h264 @ 0x555cf49ff000] error while decoding MB 83 50, bytestream -7 [in#0/rtsp @ 0x555cf4954e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555cf4954e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:13:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 223.957, "last_started_at": "2026-07-06T13:13:24+09:00", "pid": 7718, "process": {"exists": false, "pid": 7718}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.374, "sample_count": 2169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16010, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:17:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-06T13:17:07+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T13:17:07+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T13:17:08.003175+09:00", "mtime_epoch": 1783311428.0031745, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224512}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:17:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1317.867, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 78, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.601, "pushed": 2.601, "read": 2.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.873, "count": 78, "max_ms": 6.254, "p95_ms": 5.644}, "gstreamer_push": {"avg_ms": 1.449, "count": 78, "max_ms": 4.375, "p95_ms": 3.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.303, "count": 78, "max_ms": 84.809, "p95_ms": 80.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.322, "count": 78, "max_ms": 123.36, "p95_ms": 106.764}, "yolo11_640_fire_11class": {"avg_ms": 67.97, "count": 78, "max_ms": 88.324, "p95_ms": 84.091}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.718, "count": 78, "max_ms": 100.515, "p95_ms": 94.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 109.355, "count": 78, "max_ms": 161.251, "p95_ms": 157.789}, "read_interval": {"avg_ms": 384.556, "count": 78, "max_ms": 11935.901, "p95_ms": 170.56}, "triton_wait": {"avg_ms": 83.488, "count": 78, "max_ms": 123.374, "p95_ms": 106.777}, "visualize": {"avg_ms": 4.21, "count": 78, "max_ms": 6.159, "p95_ms": 5.368}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5352.867, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:17:08+09:00", "last_error_summary": "[h264 @ 0x555cf49ff000] error while decoding MB 83 50, bytestream -7 [in#0/rtsp @ 0x555cf4954e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555cf4954e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:17:08+09:00", "last_error_summary": "[h264 @ 0x555cf49ff000] error while decoding MB 83 50, bytestream -7 [in#0/rtsp @ 0x555cf4954e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555cf4954e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:17:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.0, "last_started_at": "2026-07-06T13:13:24+09:00", "pid": 7718, "process": {"exists": false, "pid": 7718}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.374, "sample_count": 2169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16011, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:17:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T13:17:08+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-06T13:17:07+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-06T13:17:08.003175+09:00", "mtime_epoch": 1783311428.0031745, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224512}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:17:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1317.875, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 77, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.606, "pushed": 2.606, "read": 2.573}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.874, "count": 78, "max_ms": 6.254, "p95_ms": 5.644}, "gstreamer_push": {"avg_ms": 1.429, "count": 78, "max_ms": 4.375, "p95_ms": 3.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.451, "count": 78, "max_ms": 84.809, "p95_ms": 80.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.64, "count": 78, "max_ms": 123.36, "p95_ms": 106.764}, "yolo11_640_fire_11class": {"avg_ms": 68.165, "count": 78, "max_ms": 88.324, "p95_ms": 84.091}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.01, "count": 78, "max_ms": 100.515, "p95_ms": 94.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 109.681, "count": 78, "max_ms": 161.251, "p95_ms": 157.789}, "read_interval": {"avg_ms": 388.447, "count": 77, "max_ms": 11935.901, "p95_ms": 170.56}, "triton_wait": {"avg_ms": 83.806, "count": 78, "max_ms": 123.374, "p95_ms": 106.777}, "visualize": {"avg_ms": 4.212, "count": 78, "max_ms": 6.159, "p95_ms": 5.368}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5352.848, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:17:08+09:00", "last_error_summary": "[h264 @ 0x555cf49ff000] error while decoding MB 83 50, bytestream -7 [in#0/rtsp @ 0x555cf4954e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555cf4954e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:17:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.007, "last_started_at": "2026-07-06T13:13:24+09:00", "pid": 7718, "process": {"exists": false, "pid": 7718}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.374, "sample_count": 2169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16011, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:17:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T13:17:08+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T13:17:07+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-06T13:17:08.003175+09:00", "mtime_epoch": 1783311428.0031745, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224512}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:17:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1317.875, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 77, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.606, "pushed": 2.606, "read": 2.572}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.874, "count": 78, "max_ms": 6.254, "p95_ms": 5.644}, "gstreamer_push": {"avg_ms": 1.429, "count": 78, "max_ms": 4.375, "p95_ms": 3.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.451, "count": 78, "max_ms": 84.809, "p95_ms": 80.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.64, "count": 78, "max_ms": 123.36, "p95_ms": 106.764}, "yolo11_640_fire_11class": {"avg_ms": 68.165, "count": 78, "max_ms": 88.324, "p95_ms": 84.091}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.01, "count": 78, "max_ms": 100.515, "p95_ms": 94.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 109.681, "count": 78, "max_ms": 161.251, "p95_ms": 157.789}, "read_interval": {"avg_ms": 388.447, "count": 77, "max_ms": 11935.901, "p95_ms": 170.56}, "triton_wait": {"avg_ms": 83.806, "count": 78, "max_ms": 123.374, "p95_ms": 106.777}, "visualize": {"avg_ms": 4.212, "count": 78, "max_ms": 6.159, "p95_ms": 5.368}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5468.977, "limit_mb": 8796093022207.996}, "detail": {"pid": 7870, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:17:08+09:00", "last_error_summary": "[h264 @ 0x555cf49ff000] error while decoding MB 83 50, bytestream -7 [in#0/rtsp @ 0x555cf4954e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555cf4954e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:17:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:17:10+09:00", "pid": 7870, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.754, "name": "ffmpeg", "pid": 7870, "rss_mb": 156.754, "state": "R (running)", "threads": 49}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.855, "sample_count": 2130, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16011, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:17:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.655, "last_frame_pushed_at": "2026-07-06T13:17:08+09:00", "last_frame_read_age_sec": 4.754, "last_frame_read_at": "2026-07-06T13:17:07+09:00", "mjpeg": {"age_sec": 4.634, "exists": true, "mtime": "2026-07-06T13:17:08.111176+09:00", "mtime_epoch": 1783311428.1111763, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224321}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:17:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1317.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 40, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.368, "pushed": 1.368, "read": 1.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.303, "count": 41, "max_ms": 5.254, "p95_ms": 5.186}, "gstreamer_push": {"avg_ms": 1.45, "count": 41, "max_ms": 3.689, "p95_ms": 1.591}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.685, "count": 41, "max_ms": 84.809, "p95_ms": 81.587}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.064, "count": 41, "max_ms": 123.36, "p95_ms": 103.001}, "yolo11_640_fire_11class": {"avg_ms": 72.902, "count": 41, "max_ms": 86.536, "p95_ms": 84.091}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.075, "count": 41, "max_ms": 100.515, "p95_ms": 94.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.428, "count": 41, "max_ms": 161.251, "p95_ms": 131.908}, "read_interval": {"avg_ms": 632.379, "count": 40, "max_ms": 11935.901, "p95_ms": 170.56}, "triton_wait": {"avg_ms": 90.14, "count": 41, "max_ms": 123.374, "p95_ms": 103.014}, "visualize": {"avg_ms": 3.857, "count": 41, "max_ms": 5.253, "p95_ms": 5.086}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 136}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5348.102, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558eb1a6d240] error while decoding MB 44 18, bytestream -5 [in#0/rtsp @ 0x558eb1a4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558eb1a4ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:18:37+09:00", "last_error_summary": "[h264 @ 0x558eb1a6d240] error while decoding MB 44 18, bytestream -5 [in#0/rtsp @ 0x558eb1a4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558eb1a4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:17:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 87.184, "last_started_at": "2026-07-06T13:17:10+09:00", "pid": 7870, "process": {"exists": false, "pid": 7870}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.287, "sample_count": 2059, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:18:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-06T13:18:37+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-06T13:18:37+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-06T13:18:37.864667+09:00", "mtime_epoch": 1783311517.8646667, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:18:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1313.895, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.679, "pushed": 0.679, "read": 0.735}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.231, "count": 12, "max_ms": 3.486, "p95_ms": 3.486}, "gstreamer_push": {"avg_ms": 1.366, "count": 12, "max_ms": 1.626, "p95_ms": 1.626}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.726, "count": 12, "max_ms": 77.497, "p95_ms": 77.497}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.826, "count": 12, "max_ms": 104.525, "p95_ms": 104.525}, "yolo11_640_fire_11class": {"avg_ms": 63.53, "count": 12, "max_ms": 88.686, "p95_ms": 88.686}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.141, "count": 12, "max_ms": 92.972, "p95_ms": 92.972}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.835, "count": 12, "max_ms": 116.981, "p95_ms": 116.981}, "read_interval": {"avg_ms": 2568.081, "count": 13, "max_ms": 16451.292, "p95_ms": 16451.292}, "triton_wait": {"avg_ms": 79.893, "count": 12, "max_ms": 104.538, "p95_ms": 104.538}, "visualize": {"avg_ms": 3.658, "count": 12, "max_ms": 3.863, "p95_ms": 3.863}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.677, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5347.25, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:18:37+09:00", "last_error_summary": "[h264 @ 0x558eb1a6d240] error while decoding MB 44 18, bytestream -5 [in#0/rtsp @ 0x558eb1a4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558eb1a4ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:18:37+09:00", "last_error_summary": "[h264 @ 0x558eb1a6d240] error while decoding MB 44 18, bytestream -5 [in#0/rtsp @ 0x558eb1a4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558eb1a4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:18:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 87.228, "last_started_at": "2026-07-06T13:17:10+09:00", "pid": 7870, "process": {"exists": false, "pid": 7870}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.287, "sample_count": 2060, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:18:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T13:18:37+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T13:18:37+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T13:18:37.864667+09:00", "mtime_epoch": 1783311517.8646667, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:18:37+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1313.895, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.734, "pushed": 0.734, "read": 0.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.261, "count": 13, "max_ms": 3.625, "p95_ms": 3.625}, "gstreamer_push": {"avg_ms": 1.359, "count": 13, "max_ms": 1.626, "p95_ms": 1.626}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.001, "count": 13, "max_ms": 77.497, "p95_ms": 77.497}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.715, "count": 13, "max_ms": 104.525, "p95_ms": 104.525}, "yolo11_640_fire_11class": {"avg_ms": 64.676, "count": 13, "max_ms": 88.686, "p95_ms": 88.686}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.912, "count": 13, "max_ms": 92.972, "p95_ms": 92.972}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.847, "count": 13, "max_ms": 116.981, "p95_ms": 116.981}, "read_interval": {"avg_ms": 2568.081, "count": 13, "max_ms": 16451.292, "p95_ms": 16451.292}, "triton_wait": {"avg_ms": 80.838, "count": 13, "max_ms": 104.538, "p95_ms": 104.538}, "visualize": {"avg_ms": 3.662, "count": 13, "max_ms": 3.863, "p95_ms": 3.863}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.721, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5347.34, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:18:37+09:00", "last_error_summary": "[h264 @ 0x558eb1a6d240] error while decoding MB 44 18, bytestream -5 [in#0/rtsp @ 0x558eb1a4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558eb1a4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:18:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 87.235, "last_started_at": "2026-07-06T13:17:10+09:00", "pid": 7870, "process": {"exists": false, "pid": 7870}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.287, "sample_count": 2059, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:18:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T13:18:37+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T13:18:37+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T13:18:37.864667+09:00", "mtime_epoch": 1783311517.8646667, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:18:37+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1313.895, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.733, "pushed": 0.733, "read": 0.733}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.261, "count": 13, "max_ms": 3.625, "p95_ms": 3.625}, "gstreamer_push": {"avg_ms": 1.359, "count": 13, "max_ms": 1.626, "p95_ms": 1.626}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.001, "count": 13, "max_ms": 77.497, "p95_ms": 77.497}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.715, "count": 13, "max_ms": 104.525, "p95_ms": 104.525}, "yolo11_640_fire_11class": {"avg_ms": 64.676, "count": 13, "max_ms": 88.686, "p95_ms": 88.686}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.912, "count": 13, "max_ms": 92.972, "p95_ms": 92.972}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.847, "count": 13, "max_ms": 116.981, "p95_ms": 116.981}, "read_interval": {"avg_ms": 2568.081, "count": 13, "max_ms": 16451.292, "p95_ms": 16451.292}, "triton_wait": {"avg_ms": 80.838, "count": 13, "max_ms": 104.538, "p95_ms": 104.538}, "visualize": {"avg_ms": 3.662, "count": 13, "max_ms": 3.863, "p95_ms": 3.863}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.727, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5522.359, "limit_mb": 8796093022207.996}, "detail": {"pid": 8022, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:18:37+09:00", "last_error_summary": "[h264 @ 0x558eb1a6d240] error while decoding MB 44 18, bytestream -5 [in#0/rtsp @ 0x558eb1a4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558eb1a4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:18:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:18:40+09:00", "pid": 8022, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.688, "name": "ffmpeg", "pid": 8022, "rss_mb": 204.688, "state": "S (sleeping)", "threads": 106}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.349, "sample_count": 2014, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:18:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 5.025, "last_frame_pushed_at": "2026-07-06T13:18:37+09:00", "last_frame_read_age_sec": 5.133, "last_frame_read_at": "2026-07-06T13:18:37+09:00", "mjpeg": {"age_sec": 5.004, "exists": true, "mtime": "2026-07-06T13:18:37.988669+09:00", "mtime_epoch": 1783311517.988669, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 212265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:18:42+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1313.895, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.572, "pushed": 0.572, "read": 0.572}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.261, "count": 13, "max_ms": 3.625, "p95_ms": 3.625}, "gstreamer_push": {"avg_ms": 1.359, "count": 13, "max_ms": 1.626, "p95_ms": 1.626}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.001, "count": 13, "max_ms": 77.497, "p95_ms": 77.497}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.715, "count": 13, "max_ms": 104.525, "p95_ms": 104.525}, "yolo11_640_fire_11class": {"avg_ms": 64.676, "count": 13, "max_ms": 88.686, "p95_ms": 88.686}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.912, "count": 13, "max_ms": 92.972, "p95_ms": 92.972}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.847, "count": 13, "max_ms": 116.981, "p95_ms": 116.981}, "read_interval": {"avg_ms": 2568.081, "count": 13, "max_ms": 16451.292, "p95_ms": 16451.292}, "triton_wait": {"avg_ms": 80.838, "count": 13, "max_ms": 104.538, "p95_ms": 104.538}, "visualize": {"avg_ms": 3.662, "count": 13, "max_ms": 3.863, "p95_ms": 3.863}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.741, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 168}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5369.754, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5579975bcfc0] error while decoding MB 31 21, bytestream -7 [h264 @ 0x557996b71c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996b71c80] error while decoding MB 0 6, bytestream -2 [h264 @ 0x557996bc5240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996bc5240] error while decoding MB 0 28, bytestream 50194 [h264 @ 0x5579974ee200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579974ee200] error while decoding MB 0 42, bytes", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:20:24+09:00", "last_error_summary": "[h264 @ 0x5579975bcfc0] error while decoding MB 31 21, bytestream -7 [h264 @ 0x557996b71c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996b71c80] error while decoding MB 0 6, bytestream -2 [h264 @ 0x557996bc5240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996bc5240] error while decoding MB 0 28, bytestream 50194 [h264 @ 0x5579974ee200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579974ee200] error while decoding MB 0 42, bytes", "last_exit_at": "2026-07-06T13:01:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 1135.996, "last_started_at": "2026-07-06T13:01:28+09:00", "pid": 7155, "process": {"exists": false, "pid": 7155}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.908, "sample_count": 2644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20193, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:20:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-06T13:20:24+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T13:20:24+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T13:20:24.338479+09:00", "mtime_epoch": 1783311624.3384788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 216953}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:20:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1443.582, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.87, "pushed": 3.87, "read": 3.87}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.265, "count": 116, "max_ms": 4.019, "p95_ms": 3.602}, "gstreamer_push": {"avg_ms": 1.533, "count": 116, "max_ms": 6.221, "p95_ms": 3.688}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.46, "count": 116, "max_ms": 66.454, "p95_ms": 60.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.184, "count": 116, "max_ms": 92.544, "p95_ms": 88.783}, "yolo11_640_fire_11class": {"avg_ms": 54.227, "count": 116, "max_ms": 65.669, "p95_ms": 63.909}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.223, "count": 116, "max_ms": 67.838, "p95_ms": 65.207}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.056, "count": 116, "max_ms": 104.531, "p95_ms": 101.079}, "read_interval": {"avg_ms": 258.483, "count": 116, "max_ms": 10089.689, "p95_ms": 115.659}, "triton_wait": {"avg_ms": 65.196, "count": 116, "max_ms": 92.557, "p95_ms": 88.796}, "visualize": {"avg_ms": 3.617, "count": 116, "max_ms": 4.695, "p95_ms": 3.968}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 168}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5369.324, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:20:24+09:00", "last_error_summary": "[h264 @ 0x5579975bcfc0] error while decoding MB 31 21, bytestream -7 [h264 @ 0x557996b71c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996b71c80] error while decoding MB 0 6, bytestream -2 [h264 @ 0x557996bc5240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996bc5240] error while decoding MB 0 28, bytestream 50194 [h264 @ 0x5579974ee200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579974ee200] error while decoding MB 0 42, bytes", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:20:24+09:00", "last_error_summary": "[h264 @ 0x5579975bcfc0] error while decoding MB 31 21, bytestream -7 [h264 @ 0x557996b71c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996b71c80] error while decoding MB 0 6, bytestream -2 [h264 @ 0x557996bc5240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996bc5240] error while decoding MB 0 28, bytestream 50194 [h264 @ 0x5579974ee200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579974ee200] error while decoding MB 0 42, bytes", "last_exit_at": "2026-07-06T13:20:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 1136.018, "last_started_at": "2026-07-06T13:01:28+09:00", "pid": 7155, "process": {"exists": false, "pid": 7155}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.908, "sample_count": 2645, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20194, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:20:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T13:20:24+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-06T13:20:24+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-06T13:20:24.338479+09:00", "mtime_epoch": 1783311624.3384788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 216953}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:20:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1443.582, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.9, "pushed": 3.9, "read": 3.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.273, "count": 116, "max_ms": 4.392, "p95_ms": 3.627}, "gstreamer_push": {"avg_ms": 1.531, "count": 117, "max_ms": 6.221, "p95_ms": 3.688}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.46, "count": 116, "max_ms": 66.454, "p95_ms": 60.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.166, "count": 116, "max_ms": 92.544, "p95_ms": 88.783}, "yolo11_640_fire_11class": {"avg_ms": 54.226, "count": 116, "max_ms": 65.669, "p95_ms": 63.909}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.223, "count": 116, "max_ms": 67.838, "p95_ms": 65.207}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.072, "count": 117, "max_ms": 104.531, "p95_ms": 101.079}, "read_interval": {"avg_ms": 258.483, "count": 116, "max_ms": 10089.689, "p95_ms": 115.659}, "triton_wait": {"avg_ms": 65.179, "count": 116, "max_ms": 92.557, "p95_ms": 88.796}, "visualize": {"avg_ms": 3.654, "count": 116, "max_ms": 7.876, "p95_ms": 3.972}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 168}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5369.254, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:20:24+09:00", "last_error_summary": "[h264 @ 0x5579975bcfc0] error while decoding MB 31 21, bytestream -7 [h264 @ 0x557996b71c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996b71c80] error while decoding MB 0 6, bytestream -2 [h264 @ 0x557996bc5240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996bc5240] error while decoding MB 0 28, bytestream 50194 [h264 @ 0x5579974ee200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579974ee200] error while decoding MB 0 42, bytes", "last_exit_at": "2026-07-06T13:20:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 1136.026, "last_started_at": "2026-07-06T13:01:28+09:00", "pid": 7155, "process": {"exists": false, "pid": 7155}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.908, "sample_count": 2645, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20194, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:20:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T13:20:24+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-06T13:20:24+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-06T13:20:24.338479+09:00", "mtime_epoch": 1783311624.3384788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 216953}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:20:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1443.582, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.868, "pushed": 3.868, "read": 3.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.273, "count": 116, "max_ms": 4.392, "p95_ms": 3.627}, "gstreamer_push": {"avg_ms": 1.511, "count": 116, "max_ms": 6.221, "p95_ms": 3.61}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.46, "count": 116, "max_ms": 66.454, "p95_ms": 60.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.166, "count": 116, "max_ms": 92.544, "p95_ms": 88.783}, "yolo11_640_fire_11class": {"avg_ms": 54.226, "count": 116, "max_ms": 65.669, "p95_ms": 63.909}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.223, "count": 116, "max_ms": 67.838, "p95_ms": 65.207}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.09, "count": 116, "max_ms": 104.531, "p95_ms": 101.079}, "read_interval": {"avg_ms": 258.483, "count": 116, "max_ms": 10089.689, "p95_ms": 115.659}, "triton_wait": {"avg_ms": 65.179, "count": 116, "max_ms": 92.557, "p95_ms": 88.796}, "visualize": {"avg_ms": 3.654, "count": 116, "max_ms": 7.876, "p95_ms": 3.972}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 142}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5084.5, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x56248d464200] missing picture in access unit with size 18 [h264 @ 0x56248e16ed00] no frame! [in#0/rtsp @ 0x56248d45fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56248d45fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56248d464840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:20:27+09:00", "last_error_summary": "[NULL @ 0x56248d464200] missing picture in access unit with size 18 [h264 @ 0x56248e16ed00] no frame! [in#0/rtsp @ 0x56248d45fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56248d45fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56248d464840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T13:18:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 106.456, "last_started_at": "2026-07-06T13:18:40+09:00", "pid": 8022, "process": {"exists": false, "pid": 8022}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.993, "sample_count": 1731, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17066, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:20:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T13:20:27+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T13:20:27+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-06T13:20:27.390532+09:00", "mtime_epoch": 1783311627.3905315, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 216637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:20:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1318.195, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.691, "pushed": 0.691, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.743, "count": 12, "max_ms": 6.29, "p95_ms": 6.29}, "gstreamer_push": {"avg_ms": 1.275, "count": 12, "max_ms": 1.455, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.854, "count": 12, "max_ms": 63.308, "p95_ms": 63.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.418, "count": 12, "max_ms": 87.843, "p95_ms": 87.843}, "yolo11_640_fire_11class": {"avg_ms": 56.271, "count": 12, "max_ms": 64.584, "p95_ms": 64.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.392, "count": 12, "max_ms": 67.791, "p95_ms": 67.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.844, "count": 12, "max_ms": 113.06, "p95_ms": 113.06}, "read_interval": {"avg_ms": 2567.482, "count": 13, "max_ms": 16238.219, "p95_ms": 16238.219}, "triton_wait": {"avg_ms": 65.49, "count": 12, "max_ms": 87.856, "p95_ms": 87.856}, "visualize": {"avg_ms": 3.844, "count": 12, "max_ms": 5.06, "p95_ms": 5.06}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.354, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 142}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5080.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:20:27+09:00", "last_error_summary": "[NULL @ 0x56248d464200] missing picture in access unit with size 18 [h264 @ 0x56248e16ed00] no frame! [in#0/rtsp @ 0x56248d45fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56248d45fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56248d464840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:20:27+09:00", "last_error_summary": "[NULL @ 0x56248d464200] missing picture in access unit with size 18 [h264 @ 0x56248e16ed00] no frame! [in#0/rtsp @ 0x56248d45fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56248d45fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56248d464840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T13:20:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 106.479, "last_started_at": "2026-07-06T13:18:40+09:00", "pid": 8022, "process": {"exists": false, "pid": 8022}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.767, "sample_count": 1732, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17067, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:20:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T13:20:27+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-06T13:20:27+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-06T13:20:27.390532+09:00", "mtime_epoch": 1783311627.3905315, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 216637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:20:27+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1318.195, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.748, "pushed": 0.748, "read": 0.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.788, "count": 13, "max_ms": 6.29, "p95_ms": 6.29}, "gstreamer_push": {"avg_ms": 1.27, "count": 13, "max_ms": 1.455, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.946, "count": 13, "max_ms": 63.308, "p95_ms": 63.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.053, "count": 13, "max_ms": 87.843, "p95_ms": 87.843}, "yolo11_640_fire_11class": {"avg_ms": 56.353, "count": 13, "max_ms": 64.584, "p95_ms": 64.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.641, "count": 13, "max_ms": 67.791, "p95_ms": 67.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.414, "count": 13, "max_ms": 113.06, "p95_ms": 113.06}, "read_interval": {"avg_ms": 2567.482, "count": 13, "max_ms": 16238.219, "p95_ms": 16238.219}, "triton_wait": {"avg_ms": 65.194, "count": 13, "max_ms": 87.856, "p95_ms": 87.856}, "visualize": {"avg_ms": 4.067, "count": 13, "max_ms": 6.739, "p95_ms": 6.739}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.376, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 142}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5080.402, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:20:27+09:00", "last_error_summary": "[NULL @ 0x56248d464200] missing picture in access unit with size 18 [h264 @ 0x56248e16ed00] no frame! [in#0/rtsp @ 0x56248d45fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56248d45fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56248d464840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T13:20:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 106.486, "last_started_at": "2026-07-06T13:18:40+09:00", "pid": 8022, "process": {"exists": false, "pid": 8022}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.767, "sample_count": 1732, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17067, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:20:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T13:20:27+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-06T13:20:27+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-06T13:20:27.390532+09:00", "mtime_epoch": 1783311627.3905315, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 216637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:20:27+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1318.195, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.748, "pushed": 0.748, "read": 0.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.788, "count": 13, "max_ms": 6.29, "p95_ms": 6.29}, "gstreamer_push": {"avg_ms": 1.27, "count": 13, "max_ms": 1.455, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.946, "count": 13, "max_ms": 63.308, "p95_ms": 63.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.053, "count": 13, "max_ms": 87.843, "p95_ms": 87.843}, "yolo11_640_fire_11class": {"avg_ms": 56.353, "count": 13, "max_ms": 64.584, "p95_ms": 64.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.641, "count": 13, "max_ms": 67.791, "p95_ms": 67.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.414, "count": 13, "max_ms": 113.06, "p95_ms": 113.06}, "read_interval": {"avg_ms": 2567.482, "count": 13, "max_ms": 16238.219, "p95_ms": 16238.219}, "triton_wait": {"avg_ms": 65.194, "count": 13, "max_ms": 87.856, "p95_ms": 87.856}, "visualize": {"avg_ms": 4.067, "count": 13, "max_ms": 6.739, "p95_ms": 6.739}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.383, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 168}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5172.344, "limit_mb": 8796093022207.996}, "detail": {"pid": 8174, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:20:24+09:00", "last_error_summary": "[h264 @ 0x5579975bcfc0] error while decoding MB 31 21, bytestream -7 [h264 @ 0x557996b71c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996b71c80] error while decoding MB 0 6, bytestream -2 [h264 @ 0x557996bc5240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557996bc5240] error while decoding MB 0 28, bytestream 50194 [h264 @ 0x5579974ee200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579974ee200] error while decoding MB 0 42, bytes", "last_exit_at": "2026-07-06T13:20:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:20:27+09:00", "pid": 8174, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.324, "name": "ffmpeg", "pid": 8174, "rss_mb": 136.324, "state": "R (running)", "threads": 106}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.586, "sample_count": 2603, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20194, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:20:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.682, "last_frame_pushed_at": "2026-07-06T13:20:24+09:00", "last_frame_read_age_sec": 4.763, "last_frame_read_at": "2026-07-06T13:20:24+09:00", "mjpeg": {"age_sec": 4.664, "exists": true, "mtime": "2026-07-06T13:20:24.430480+09:00", "mtime_epoch": 1783311624.4304805, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 218853}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:20:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1443.582, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.1, "pushed": 2.1, "read": 2.1}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.272, "count": 63, "max_ms": 4.392, "p95_ms": 3.769}, "gstreamer_push": {"avg_ms": 1.558, "count": 63, "max_ms": 6.052, "p95_ms": 3.61}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.314, "count": 63, "max_ms": 66.454, "p95_ms": 64.622}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.09, "count": 63, "max_ms": 92.544, "p95_ms": 89.313}, "yolo11_640_fire_11class": {"avg_ms": 55.361, "count": 63, "max_ms": 65.669, "p95_ms": 64.793}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.363, "count": 63, "max_ms": 67.838, "p95_ms": 65.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.043, "count": 63, "max_ms": 104.531, "p95_ms": 101.181}, "read_interval": {"avg_ms": 401.811, "count": 63, "max_ms": 10089.689, "p95_ms": 116.761}, "triton_wait": {"avg_ms": 67.102, "count": 63, "max_ms": 92.557, "p95_ms": 89.325}, "visualize": {"avg_ms": 3.662, "count": 63, "max_ms": 7.876, "p95_ms": 3.972}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 142}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5352.43, "limit_mb": 8796093022207.996}, "detail": {"pid": 8281, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:20:27+09:00", "last_error_summary": "[NULL @ 0x56248d464200] missing picture in access unit with size 18 [h264 @ 0x56248e16ed00] no frame! [in#0/rtsp @ 0x56248d45fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56248d45fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56248d464840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T13:20:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:20:29+09:00", "pid": 8281, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.855, "name": "ffmpeg", "pid": 8281, "rss_mb": 156.691, "state": "R (running)", "threads": 49}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.993, "sample_count": 1695, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17067, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:20:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.388, "last_frame_pushed_at": "2026-07-06T13:20:27+09:00", "last_frame_read_age_sec": 4.468, "last_frame_read_at": "2026-07-06T13:20:27+09:00", "mjpeg": {"age_sec": 4.371, "exists": true, "mtime": "2026-07-06T13:20:27.478533+09:00", "mtime_epoch": 1783311627.478533, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 215629}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:20:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1318.195, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.598, "pushed": 0.598, "read": 0.598}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.788, "count": 13, "max_ms": 6.29, "p95_ms": 6.29}, "gstreamer_push": {"avg_ms": 1.27, "count": 13, "max_ms": 1.455, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.946, "count": 13, "max_ms": 63.308, "p95_ms": 63.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.053, "count": 13, "max_ms": 87.843, "p95_ms": 87.843}, "yolo11_640_fire_11class": {"avg_ms": 56.353, "count": 13, "max_ms": 64.584, "p95_ms": 64.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.641, "count": 13, "max_ms": 67.791, "p95_ms": 67.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.414, "count": 13, "max_ms": 113.06, "p95_ms": 113.06}, "read_interval": {"avg_ms": 2567.482, "count": 13, "max_ms": 16238.219, "p95_ms": 16238.219}, "triton_wait": {"avg_ms": 65.194, "count": 13, "max_ms": 87.856, "p95_ms": 87.856}, "visualize": {"avg_ms": 4.067, "count": 13, "max_ms": 6.739, "p95_ms": 6.739}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.756, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5326.008, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555bdf219940] error while decoding MB 0 24, bytestream -9 [in#0/rtsp @ 0x555bdf076e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555bdf076e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:22:47+09:00", "last_error_summary": "[h264 @ 0x555bdf219940] error while decoding MB 0 24, bytestream -9 [in#0/rtsp @ 0x555bdf076e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555bdf076e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:20:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.953, "last_started_at": "2026-07-06T13:20:29+09:00", "pid": 8281, "process": {"exists": false, "pid": 8281}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.521, "sample_count": 1713, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17995, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:22:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T13:22:47+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-06T13:22:47+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-06T13:22:47.756980+09:00", "mtime_epoch": 1783311767.7569804, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 209100}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:22:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1314.289, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.685, "pushed": 0.685, "read": 0.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.939, "count": 12, "max_ms": 3.443, "p95_ms": 3.443}, "gstreamer_push": {"avg_ms": 1.414, "count": 12, "max_ms": 3.541, "p95_ms": 3.541}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.596, "count": 12, "max_ms": 66.65, "p95_ms": 66.65}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.988, "count": 12, "max_ms": 100.157, "p95_ms": 100.157}, "yolo11_640_fire_11class": {"avg_ms": 58.504, "count": 12, "max_ms": 74.81, "p95_ms": 74.81}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.945, "count": 12, "max_ms": 82.13, "p95_ms": 82.13}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.026, "count": 12, "max_ms": 111.767, "p95_ms": 111.767}, "read_interval": {"avg_ms": 2555.416, "count": 13, "max_ms": 16377.389, "p95_ms": 16377.389}, "triton_wait": {"avg_ms": 75.0, "count": 12, "max_ms": 100.171, "p95_ms": 100.171}, "visualize": {"avg_ms": 3.382, "count": 12, "max_ms": 4.09, "p95_ms": 4.09}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.509, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5325.93, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:22:47+09:00", "last_error_summary": "[h264 @ 0x555bdf219940] error while decoding MB 0 24, bytestream -9 [in#0/rtsp @ 0x555bdf076e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555bdf076e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:22:47+09:00", "last_error_summary": "[h264 @ 0x555bdf219940] error while decoding MB 0 24, bytestream -9 [in#0/rtsp @ 0x555bdf076e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555bdf076e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:22:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.979, "last_started_at": "2026-07-06T13:20:29+09:00", "pid": 8281, "process": {"exists": false, "pid": 8281}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.521, "sample_count": 1714, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17996, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:22:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T13:22:47+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-06T13:22:47+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-06T13:22:47.756980+09:00", "mtime_epoch": 1783311767.7569804, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 209100}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:22:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1314.289, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.741, "pushed": 0.741, "read": 0.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.989, "count": 13, "max_ms": 3.588, "p95_ms": 3.588}, "gstreamer_push": {"avg_ms": 1.402, "count": 13, "max_ms": 3.541, "p95_ms": 3.541}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.825, "count": 13, "max_ms": 66.65, "p95_ms": 66.65}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.875, "count": 13, "max_ms": 100.157, "p95_ms": 100.157}, "yolo11_640_fire_11class": {"avg_ms": 58.283, "count": 13, "max_ms": 74.81, "p95_ms": 74.81}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.427, "count": 13, "max_ms": 82.13, "p95_ms": 82.13}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.039, "count": 13, "max_ms": 111.767, "p95_ms": 111.767}, "read_interval": {"avg_ms": 2555.416, "count": 13, "max_ms": 16377.389, "p95_ms": 16377.389}, "triton_wait": {"avg_ms": 73.887, "count": 13, "max_ms": 100.171, "p95_ms": 100.171}, "visualize": {"avg_ms": 3.455, "count": 13, "max_ms": 4.326, "p95_ms": 4.326}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.535, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5325.922, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:22:47+09:00", "last_error_summary": "[h264 @ 0x555bdf219940] error while decoding MB 0 24, bytestream -9 [in#0/rtsp @ 0x555bdf076e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555bdf076e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:22:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.985, "last_started_at": "2026-07-06T13:20:29+09:00", "pid": 8281, "process": {"exists": false, "pid": 8281}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.521, "sample_count": 1714, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17996, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:22:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T13:22:47+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-06T13:22:47+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-06T13:22:47.756980+09:00", "mtime_epoch": 1783311767.7569804, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 209100}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:22:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1314.289, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.741, "pushed": 0.741, "read": 0.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.989, "count": 13, "max_ms": 3.588, "p95_ms": 3.588}, "gstreamer_push": {"avg_ms": 1.402, "count": 13, "max_ms": 3.541, "p95_ms": 3.541}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.825, "count": 13, "max_ms": 66.65, "p95_ms": 66.65}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.875, "count": 13, "max_ms": 100.157, "p95_ms": 100.157}, "yolo11_640_fire_11class": {"avg_ms": 58.283, "count": 13, "max_ms": 74.81, "p95_ms": 74.81}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.427, "count": 13, "max_ms": 82.13, "p95_ms": 82.13}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.039, "count": 13, "max_ms": 111.767, "p95_ms": 111.767}, "read_interval": {"avg_ms": 2555.416, "count": 13, "max_ms": 16377.389, "p95_ms": 16377.389}, "triton_wait": {"avg_ms": 73.887, "count": 13, "max_ms": 100.171, "p95_ms": 100.171}, "visualize": {"avg_ms": 3.455, "count": 13, "max_ms": 4.326, "p95_ms": 4.326}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.541, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5342.789, "limit_mb": 8796093022207.996}, "detail": {"pid": 8433, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:22:47+09:00", "last_error_summary": "[h264 @ 0x555bdf219940] error while decoding MB 0 24, bytestream -9 [in#0/rtsp @ 0x555bdf076e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555bdf076e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:22:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:22:50+09:00", "pid": 8433, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.477, "name": "ffmpeg", "pid": 8433, "rss_mb": 58.477, "state": "S (sleeping)", "threads": 1}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.968, "sample_count": 1675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17996, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:22:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.557, "last_frame_pushed_at": "2026-07-06T13:22:47+09:00", "last_frame_read_age_sec": 4.632, "last_frame_read_at": "2026-07-06T13:22:47+09:00", "mjpeg": {"age_sec": 4.54, "exists": true, "mtime": "2026-07-06T13:22:47.836982+09:00", "mtime_epoch": 1783311767.8369818, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 208021}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:22:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.109, "name": "celery", "pid": 267, "rss_mb": 1314.289, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.589, "pushed": 0.589, "read": 0.589}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.989, "count": 13, "max_ms": 3.588, "p95_ms": 3.588}, "gstreamer_push": {"avg_ms": 1.402, "count": 13, "max_ms": 3.541, "p95_ms": 3.541}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.825, "count": 13, "max_ms": 66.65, "p95_ms": 66.65}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.875, "count": 13, "max_ms": 100.157, "p95_ms": 100.157}, "yolo11_640_fire_11class": {"avg_ms": 58.283, "count": 13, "max_ms": 74.81, "p95_ms": 74.81}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.427, "count": 13, "max_ms": 82.13, "p95_ms": 82.13}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.039, "count": 13, "max_ms": 111.767, "p95_ms": 111.767}, "read_interval": {"avg_ms": 2555.416, "count": 13, "max_ms": 16377.389, "p95_ms": 16377.389}, "triton_wait": {"avg_ms": 73.887, "count": 13, "max_ms": 100.171, "p95_ms": 100.171}, "visualize": {"avg_ms": 3.455, "count": 13, "max_ms": 4.326, "p95_ms": 4.326}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.084, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 195}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5393.082, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5620a8de97c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5620a8de97c0] error while decoding MB 0 24, bytestream 1774 [h264 @ 0x5620a83236c0] reference picture missing during reorder [h264 @ 0x5620a83236c0] Missing reference picture, default is 65576 [h264 @ 0x5620a82d0100] mmco: unref short failure [h264 @ 0x5620a82d0100] illegal short term buffer state detected [h264 @ 0x5620a8366800] reference picture missing during reorder [h264 @ 0x5620a8366800] Missing referen", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:26:57+09:00", "last_error_summary": "[h264 @ 0x5620a8de97c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5620a8de97c0] error while decoding MB 0 24, bytestream 1774 [h264 @ 0x5620a83236c0] reference picture missing during reorder [h264 @ 0x5620a83236c0] Missing reference picture, default is 65576 [h264 @ 0x5620a82d0100] mmco: unref short failure [h264 @ 0x5620a82d0100] illegal short term buffer state detected [h264 @ 0x5620a8366800] reference picture missing during reorder [h264 @ 0x5620a8366800] Missing referen", "last_exit_at": "2026-07-06T13:20:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 390.584, "last_started_at": "2026-07-06T13:20:27+09:00", "pid": 8174, "process": {"exists": false, "pid": 8174}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 120.314, "sample_count": 2567, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23516, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:26:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-06T13:26:57+09:00", "last_frame_read_age_sec": 0.043, "last_frame_read_at": "2026-07-06T13:26:57+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-06T13:26:57.629460+09:00", "mtime_epoch": 1783312017.6294599, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 224802}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:26:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1450.211, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 92, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.074, "pushed": 3.074, "read": 3.074}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.358, "count": 92, "max_ms": 5.202, "p95_ms": 3.912}, "gstreamer_push": {"avg_ms": 1.526, "count": 92, "max_ms": 6.4, "p95_ms": 3.662}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.733, "count": 92, "max_ms": 86.329, "p95_ms": 82.474}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.125, "count": 92, "max_ms": 118.166, "p95_ms": 102.046}, "yolo11_640_fire_11class": {"avg_ms": 71.005, "count": 92, "max_ms": 90.041, "p95_ms": 85.716}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.92, "count": 92, "max_ms": 102.264, "p95_ms": 96.306}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.889, "count": 92, "max_ms": 132.127, "p95_ms": 114.669}, "read_interval": {"avg_ms": 325.791, "count": 92, "max_ms": 10127.33, "p95_ms": 134.458}, "triton_wait": {"avg_ms": 87.643, "count": 92, "max_ms": 118.178, "p95_ms": 102.058}, "visualize": {"avg_ms": 3.714, "count": 92, "max_ms": 4.663, "p95_ms": 4.127}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 195}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5392.32, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:26:57+09:00", "last_error_summary": "[h264 @ 0x5620a8de97c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5620a8de97c0] error while decoding MB 0 24, bytestream 1774 [h264 @ 0x5620a83236c0] reference picture missing during reorder [h264 @ 0x5620a83236c0] Missing reference picture, default is 65576 [h264 @ 0x5620a82d0100] mmco: unref short failure [h264 @ 0x5620a82d0100] illegal short term buffer state detected [h264 @ 0x5620a8366800] reference picture missing during reorder [h264 @ 0x5620a8366800] Missing referen", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:26:57+09:00", "last_error_summary": "[h264 @ 0x5620a8de97c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5620a8de97c0] error while decoding MB 0 24, bytestream 1774 [h264 @ 0x5620a83236c0] reference picture missing during reorder [h264 @ 0x5620a83236c0] Missing reference picture, default is 65576 [h264 @ 0x5620a82d0100] mmco: unref short failure [h264 @ 0x5620a82d0100] illegal short term buffer state detected [h264 @ 0x5620a8366800] reference picture missing during reorder [h264 @ 0x5620a8366800] Missing referen", "last_exit_at": "2026-07-06T13:26:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 390.65, "last_started_at": "2026-07-06T13:20:27+09:00", "pid": 8174, "process": {"exists": false, "pid": 8174}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 120.314, "sample_count": 2567, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23517, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:26:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T13:26:57+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T13:26:57+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T13:26:57.629460+09:00", "mtime_epoch": 1783312017.6294599, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 224802}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:26:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1450.211, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.101, "pushed": 3.101, "read": 3.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.357, "count": 93, "max_ms": 5.202, "p95_ms": 3.912}, "gstreamer_push": {"avg_ms": 1.525, "count": 93, "max_ms": 6.4, "p95_ms": 3.662}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.813, "count": 93, "max_ms": 86.329, "p95_ms": 82.474}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.128, "count": 93, "max_ms": 118.166, "p95_ms": 102.046}, "yolo11_640_fire_11class": {"avg_ms": 71.0, "count": 93, "max_ms": 90.041, "p95_ms": 85.716}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.938, "count": 93, "max_ms": 102.264, "p95_ms": 96.306}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.889, "count": 93, "max_ms": 132.127, "p95_ms": 114.669}, "read_interval": {"avg_ms": 325.791, "count": 92, "max_ms": 10127.33, "p95_ms": 134.458}, "triton_wait": {"avg_ms": 87.641, "count": 93, "max_ms": 118.178, "p95_ms": 102.058}, "visualize": {"avg_ms": 3.715, "count": 93, "max_ms": 4.663, "p95_ms": 4.127}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 195}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5392.184, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:26:57+09:00", "last_error_summary": "[h264 @ 0x5620a8de97c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5620a8de97c0] error while decoding MB 0 24, bytestream 1774 [h264 @ 0x5620a83236c0] reference picture missing during reorder [h264 @ 0x5620a83236c0] Missing reference picture, default is 65576 [h264 @ 0x5620a82d0100] mmco: unref short failure [h264 @ 0x5620a82d0100] illegal short term buffer state detected [h264 @ 0x5620a8366800] reference picture missing during reorder [h264 @ 0x5620a8366800] Missing referen", "last_exit_at": "2026-07-06T13:26:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 390.657, "last_started_at": "2026-07-06T13:20:27+09:00", "pid": 8174, "process": {"exists": false, "pid": 8174}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 120.314, "sample_count": 2567, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23517, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:26:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T13:26:57+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T13:26:57+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-06T13:26:57.629460+09:00", "mtime_epoch": 1783312017.6294599, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 224802}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:26:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1450.211, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.067, "pushed": 3.1, "read": 3.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.358, "count": 92, "max_ms": 5.202, "p95_ms": 3.912}, "gstreamer_push": {"avg_ms": 1.525, "count": 93, "max_ms": 6.4, "p95_ms": 3.662}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.941, "count": 92, "max_ms": 86.329, "p95_ms": 82.474}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.256, "count": 92, "max_ms": 118.166, "p95_ms": 102.046}, "yolo11_640_fire_11class": {"avg_ms": 70.987, "count": 92, "max_ms": 90.041, "p95_ms": 85.716}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.009, "count": 92, "max_ms": 102.264, "p95_ms": 96.306}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.021, "count": 92, "max_ms": 132.127, "p95_ms": 114.669}, "read_interval": {"avg_ms": 325.791, "count": 92, "max_ms": 10127.33, "p95_ms": 134.458}, "triton_wait": {"avg_ms": 87.774, "count": 92, "max_ms": 118.178, "p95_ms": 102.058}, "visualize": {"avg_ms": 3.711, "count": 92, "max_ms": 4.663, "p95_ms": 4.127}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 195}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5500.602, "limit_mb": 8796093022207.996}, "detail": {"pid": 8585, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:26:57+09:00", "last_error_summary": "[h264 @ 0x5620a8de97c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5620a8de97c0] error while decoding MB 0 24, bytestream 1774 [h264 @ 0x5620a83236c0] reference picture missing during reorder [h264 @ 0x5620a83236c0] Missing reference picture, default is 65576 [h264 @ 0x5620a82d0100] mmco: unref short failure [h264 @ 0x5620a82d0100] illegal short term buffer state detected [h264 @ 0x5620a8366800] reference picture missing during reorder [h264 @ 0x5620a8366800] Missing referen", "last_exit_at": "2026-07-06T13:27:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:27:00+09:00", "pid": 8585, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.133, "name": "ffmpeg", "pid": 8585, "rss_mb": 139.133, "state": "S (sleeping)", "threads": 106}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 120.314, "sample_count": 2528, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23517, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:26:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.566, "last_frame_pushed_at": "2026-07-06T13:26:57+09:00", "last_frame_read_age_sec": 4.668, "last_frame_read_at": "2026-07-06T13:26:57+09:00", "mjpeg": {"age_sec": 4.545, "exists": true, "mtime": "2026-07-06T13:26:57.757462+09:00", "mtime_epoch": 1783312017.7574623, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 225279}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:27:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1450.211, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.705, "pushed": 1.705, "read": 1.671}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.35, "count": 51, "max_ms": 4.931, "p95_ms": 3.912}, "gstreamer_push": {"avg_ms": 1.42, "count": 51, "max_ms": 3.968, "p95_ms": 1.545}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.02, "count": 51, "max_ms": 86.329, "p95_ms": 82.474}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 89.437, "count": 51, "max_ms": 114.298, "p95_ms": 102.676}, "yolo11_640_fire_11class": {"avg_ms": 73.363, "count": 51, "max_ms": 90.041, "p95_ms": 86.813}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.52, "count": 51, "max_ms": 102.264, "p95_ms": 98.039}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.584, "count": 51, "max_ms": 126.458, "p95_ms": 114.901}, "read_interval": {"avg_ms": 507.224, "count": 50, "max_ms": 10127.33, "p95_ms": 139.068}, "triton_wait": {"avg_ms": 90.308, "count": 51, "max_ms": 114.311, "p95_ms": 102.688}, "visualize": {"avg_ms": 3.729, "count": 51, "max_ms": 4.663, "p95_ms": 4.158}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 176}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5333.203, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a087f60600] error while decoding MB 58 48, bytestream -33 [in#0/rtsp @ 0x55a087da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a087da3e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:29:06+09:00", "last_error_summary": "[h264 @ 0x55a087f60600] error while decoding MB 58 48, bytestream -33 [in#0/rtsp @ 0x55a087da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a087da3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:22:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 376.452, "last_started_at": "2026-07-06T13:22:50+09:00", "pid": 8433, "process": {"exists": false, "pid": 8433}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 126.502, "sample_count": 2492, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:29:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T13:29:06+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T13:29:06+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T13:29:06.775818+09:00", "mtime_epoch": 1783312146.7758179, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232769}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:29:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1323.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 72, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.367, "pushed": 2.367, "read": 2.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.712, "count": 71, "max_ms": 5.857, "p95_ms": 5.722}, "gstreamer_push": {"avg_ms": 1.398, "count": 71, "max_ms": 3.79, "p95_ms": 1.642}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.929, "count": 71, "max_ms": 90.959, "p95_ms": 79.856}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.01, "count": 71, "max_ms": 125.595, "p95_ms": 103.513}, "yolo11_640_fire_11class": {"avg_ms": 69.053, "count": 71, "max_ms": 91.497, "p95_ms": 83.023}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.19, "count": 71, "max_ms": 107.159, "p95_ms": 96.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.632, "count": 71, "max_ms": 144.26, "p95_ms": 138.208}, "read_interval": {"avg_ms": 417.831, "count": 72, "max_ms": 12858.779, "p95_ms": 153.811}, "triton_wait": {"avg_ms": 86.149, "count": 71, "max_ms": 125.61, "p95_ms": 103.525}, "visualize": {"avg_ms": 3.814, "count": 71, "max_ms": 5.285, "p95_ms": 5.18}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 176}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5332.316, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:29:06+09:00", "last_error_summary": "[h264 @ 0x55a087f60600] error while decoding MB 58 48, bytestream -33 [in#0/rtsp @ 0x55a087da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a087da3e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:29:06+09:00", "last_error_summary": "[h264 @ 0x55a087f60600] error while decoding MB 58 48, bytestream -33 [in#0/rtsp @ 0x55a087da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a087da3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:29:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 376.514, "last_started_at": "2026-07-06T13:22:50+09:00", "pid": 8433, "process": {"exists": false, "pid": 8433}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 126.502, "sample_count": 2493, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21146, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:29:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T13:29:06+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T13:29:06+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T13:29:06.775818+09:00", "mtime_epoch": 1783312146.7758179, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232769}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:29:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1323.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.403, "pushed": 2.403, "read": 2.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.707, "count": 72, "max_ms": 5.857, "p95_ms": 5.722}, "gstreamer_push": {"avg_ms": 1.399, "count": 72, "max_ms": 3.79, "p95_ms": 1.642}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.971, "count": 72, "max_ms": 90.959, "p95_ms": 79.856}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.267, "count": 72, "max_ms": 125.595, "p95_ms": 104.493}, "yolo11_640_fire_11class": {"avg_ms": 69.257, "count": 72, "max_ms": 91.497, "p95_ms": 83.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.381, "count": 72, "max_ms": 107.159, "p95_ms": 96.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.811, "count": 72, "max_ms": 144.26, "p95_ms": 138.208}, "read_interval": {"avg_ms": 421.565, "count": 71, "max_ms": 12858.779, "p95_ms": 153.811}, "triton_wait": {"avg_ms": 86.404, "count": 72, "max_ms": 125.61, "p95_ms": 104.506}, "visualize": {"avg_ms": 3.812, "count": 72, "max_ms": 5.285, "p95_ms": 5.18}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 176}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5332.293, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:29:06+09:00", "last_error_summary": "[h264 @ 0x55a087f60600] error while decoding MB 58 48, bytestream -33 [in#0/rtsp @ 0x55a087da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a087da3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:29:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 376.521, "last_started_at": "2026-07-06T13:22:50+09:00", "pid": 8433, "process": {"exists": false, "pid": 8433}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 126.502, "sample_count": 2493, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21146, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:29:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T13:29:06+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T13:29:06+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T13:29:06.775818+09:00", "mtime_epoch": 1783312146.7758179, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232769}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:29:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1323.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.402, "pushed": 2.402, "read": 2.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.707, "count": 72, "max_ms": 5.857, "p95_ms": 5.722}, "gstreamer_push": {"avg_ms": 1.399, "count": 72, "max_ms": 3.79, "p95_ms": 1.642}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.971, "count": 72, "max_ms": 90.959, "p95_ms": 79.856}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.267, "count": 72, "max_ms": 125.595, "p95_ms": 104.493}, "yolo11_640_fire_11class": {"avg_ms": 69.257, "count": 72, "max_ms": 91.497, "p95_ms": 83.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.381, "count": 72, "max_ms": 107.159, "p95_ms": 96.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.811, "count": 72, "max_ms": 144.26, "p95_ms": 138.208}, "read_interval": {"avg_ms": 421.565, "count": 71, "max_ms": 12858.779, "p95_ms": 153.811}, "triton_wait": {"avg_ms": 86.404, "count": 72, "max_ms": 125.61, "p95_ms": 104.506}, "visualize": {"avg_ms": 3.812, "count": 72, "max_ms": 5.285, "p95_ms": 5.18}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 176}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5511.984, "limit_mb": 8796093022207.996}, "detail": {"pid": 8692, "restart_count": 10, "start_count": 11}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:29:06+09:00", "last_error_summary": "[h264 @ 0x55a087f60600] error while decoding MB 58 48, bytestream -33 [in#0/rtsp @ 0x55a087da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a087da3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:29:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:29:09+09:00", "pid": 8692, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.668, "name": "ffmpeg", "pid": 8692, "rss_mb": 206.668, "state": "S (sleeping)", "threads": 106}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.552, "sample_count": 2457, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21146, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:29:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.122, "last_frame_pushed_at": "2026-07-06T13:29:06+09:00", "last_frame_read_age_sec": 4.241, "last_frame_read_at": "2026-07-06T13:29:06+09:00", "mjpeg": {"age_sec": 4.101, "exists": true, "mtime": "2026-07-06T13:29:06.903820+09:00", "mtime_epoch": 1783312146.9038203, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232230}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:29:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1323.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 37, "pushed_frames": 37, "read_frames": 36, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.235, "pushed": 1.235, "read": 1.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.778, "count": 37, "max_ms": 5.857, "p95_ms": 5.728}, "gstreamer_push": {"avg_ms": 1.291, "count": 37, "max_ms": 1.57, "p95_ms": 1.544}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.481, "count": 37, "max_ms": 83.01, "p95_ms": 78.044}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.624, "count": 37, "max_ms": 125.595, "p95_ms": 104.493}, "yolo11_640_fire_11class": {"avg_ms": 68.885, "count": 37, "max_ms": 83.784, "p95_ms": 83.023}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.865, "count": 37, "max_ms": 96.149, "p95_ms": 94.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.85, "count": 37, "max_ms": 144.26, "p95_ms": 143.339}, "read_interval": {"avg_ms": 717.298, "count": 36, "max_ms": 12858.779, "p95_ms": 9002.277}, "triton_wait": {"avg_ms": 84.85, "count": 37, "max_ms": 125.61, "p95_ms": 104.506}, "visualize": {"avg_ms": 3.813, "count": 37, "max_ms": 5.285, "p95_ms": 5.264}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5363.719, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d7d1b5db00] error while decoding MB 55 25, bytestream -7 [in#0/rtsp @ 0x55d7d132de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d7d132de00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:31:18+09:00", "last_error_summary": "[h264 @ 0x55d7d1b5db00] error while decoding MB 55 25, bytestream -7 [in#0/rtsp @ 0x55d7d132de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d7d132de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:29:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.294, "last_started_at": "2026-07-06T13:29:09+09:00", "pid": 8692, "process": {"exists": false, "pid": 8692}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.951, "sample_count": 2175, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21972, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:31:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.05, "last_frame_pushed_at": "2026-07-06T13:31:18+09:00", "last_frame_read_age_sec": 0.041, "last_frame_read_at": "2026-07-06T13:31:18+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-06T13:31:18.266240+09:00", "mtime_epoch": 1783312278.2662401, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 225847}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:31:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1317.234, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.725, "pushed": 0.725, "read": 0.78}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.289, "count": 13, "max_ms": 3.657, "p95_ms": 3.657}, "gstreamer_push": {"avg_ms": 1.556, "count": 13, "max_ms": 3.812, "p95_ms": 3.812}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.765, "count": 13, "max_ms": 84.9, "p95_ms": 84.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.982, "count": 13, "max_ms": 109.043, "p95_ms": 109.043}, "yolo11_640_fire_11class": {"avg_ms": 66.052, "count": 13, "max_ms": 85.419, "p95_ms": 85.419}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.666, "count": 13, "max_ms": 93.87, "p95_ms": 93.87}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.028, "count": 13, "max_ms": 120.869, "p95_ms": 120.869}, "read_interval": {"avg_ms": 2393.602, "count": 14, "max_ms": 16625.134, "p95_ms": 16625.134}, "triton_wait": {"avg_ms": 79.994, "count": 13, "max_ms": 109.055, "p95_ms": 109.055}, "visualize": {"avg_ms": 3.64, "count": 13, "max_ms": 3.717, "p95_ms": 3.717}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.938, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5362.883, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:31:18+09:00", "last_error_summary": "[h264 @ 0x55d7d1b5db00] error while decoding MB 55 25, bytestream -7 [in#0/rtsp @ 0x55d7d132de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d7d132de00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:31:18+09:00", "last_error_summary": "[h264 @ 0x55d7d1b5db00] error while decoding MB 55 25, bytestream -7 [in#0/rtsp @ 0x55d7d132de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d7d132de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:31:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.353, "last_started_at": "2026-07-06T13:29:09+09:00", "pid": 8692, "process": {"exists": false, "pid": 8692}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.951, "sample_count": 2175, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21973, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:31:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T13:31:18+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-06T13:31:18+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-06T13:31:18.266240+09:00", "mtime_epoch": 1783312278.2662401, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 225847}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:31:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1317.234, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.778, "pushed": 0.778, "read": 0.778}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.287, "count": 14, "max_ms": 3.657, "p95_ms": 3.657}, "gstreamer_push": {"avg_ms": 1.544, "count": 14, "max_ms": 3.812, "p95_ms": 3.812}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.178, "count": 14, "max_ms": 84.9, "p95_ms": 84.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.099, "count": 14, "max_ms": 109.043, "p95_ms": 109.043}, "yolo11_640_fire_11class": {"avg_ms": 65.98, "count": 14, "max_ms": 85.419, "p95_ms": 85.419}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.034, "count": 14, "max_ms": 93.87, "p95_ms": 93.87}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.132, "count": 14, "max_ms": 120.869, "p95_ms": 120.869}, "read_interval": {"avg_ms": 2393.602, "count": 14, "max_ms": 16625.134, "p95_ms": 16625.134}, "triton_wait": {"avg_ms": 80.112, "count": 14, "max_ms": 109.055, "p95_ms": 109.055}, "visualize": {"avg_ms": 3.636, "count": 14, "max_ms": 3.717, "p95_ms": 3.717}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5362.863, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:31:18+09:00", "last_error_summary": "[h264 @ 0x55d7d1b5db00] error while decoding MB 55 25, bytestream -7 [in#0/rtsp @ 0x55d7d132de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d7d132de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:31:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.361, "last_started_at": "2026-07-06T13:29:09+09:00", "pid": 8692, "process": {"exists": false, "pid": 8692}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.951, "sample_count": 2175, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21973, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:31:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T13:31:18+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T13:31:18+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-06T13:31:18.266240+09:00", "mtime_epoch": 1783312278.2662401, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 225847}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:31:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1317.234, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.778, "pushed": 0.778, "read": 0.778}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.287, "count": 14, "max_ms": 3.657, "p95_ms": 3.657}, "gstreamer_push": {"avg_ms": 1.544, "count": 14, "max_ms": 3.812, "p95_ms": 3.812}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.178, "count": 14, "max_ms": 84.9, "p95_ms": 84.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.099, "count": 14, "max_ms": 109.043, "p95_ms": 109.043}, "yolo11_640_fire_11class": {"avg_ms": 65.98, "count": 14, "max_ms": 85.419, "p95_ms": 85.419}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.034, "count": 14, "max_ms": 93.87, "p95_ms": 93.87}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.132, "count": 14, "max_ms": 120.869, "p95_ms": 120.869}, "read_interval": {"avg_ms": 2393.602, "count": 14, "max_ms": 16625.134, "p95_ms": 16625.134}, "triton_wait": {"avg_ms": 80.112, "count": 14, "max_ms": 109.055, "p95_ms": 109.055}, "visualize": {"avg_ms": 3.636, "count": 14, "max_ms": 3.717, "p95_ms": 3.717}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.005, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5480.453, "limit_mb": 8796093022207.996}, "detail": {"pid": 8844, "restart_count": 11, "start_count": 12}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:31:18+09:00", "last_error_summary": "[h264 @ 0x55d7d1b5db00] error while decoding MB 55 25, bytestream -7 [in#0/rtsp @ 0x55d7d132de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d7d132de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:31:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:31:20+09:00", "pid": 8844, "process": {"exists": true, "fd_count": 5, "hwm_mb": 148.504, "name": "ffmpeg", "pid": 8844, "rss_mb": 148.504, "state": "R (running)", "threads": 49}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.021, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21973, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:31:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.377, "last_frame_pushed_at": "2026-07-06T13:31:18+09:00", "last_frame_read_age_sec": 4.473, "last_frame_read_at": "2026-07-06T13:31:18+09:00", "mjpeg": {"age_sec": 4.36, "exists": true, "mtime": "2026-07-06T13:31:18.370242+09:00", "mtime_epoch": 1783312278.3702419, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:31:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1317.234, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.626, "pushed": 0.626, "read": 0.626}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.287, "count": 14, "max_ms": 3.657, "p95_ms": 3.657}, "gstreamer_push": {"avg_ms": 1.544, "count": 14, "max_ms": 3.812, "p95_ms": 3.812}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.178, "count": 14, "max_ms": 84.9, "p95_ms": 84.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.099, "count": 14, "max_ms": 109.043, "p95_ms": 109.043}, "yolo11_640_fire_11class": {"avg_ms": 65.98, "count": 14, "max_ms": 85.419, "p95_ms": 85.419}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.034, "count": 14, "max_ms": 93.87, "p95_ms": 93.87}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.132, "count": 14, "max_ms": 120.869, "p95_ms": 120.869}, "read_interval": {"avg_ms": 2393.602, "count": 14, "max_ms": 16625.134, "p95_ms": 16625.134}, "triton_wait": {"avg_ms": 80.112, "count": 14, "max_ms": 109.055, "p95_ms": 109.055}, "visualize": {"avg_ms": 3.636, "count": 14, "max_ms": 3.717, "p95_ms": 3.717}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.37, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5380.043, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5587f5c93b00] error while decoding MB 77 43, bytestream -14 [h264 @ 0x5587f503aa00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f503aa00] error while decoding MB 0 26, bytestream 60127 [h264 @ 0x5587f4f3c9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f4f3c9c0] error while decoding MB 0 36, bytestream 111096 [h264 @ 0x5587f5a26f80] error while decoding MB 16 34, bytestream -5 [h264 @ 0x5587f5bc4c80] left block unavailable for requested i", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:31:57+09:00", "last_error_summary": "[h264 @ 0x5587f5c93b00] error while decoding MB 77 43, bytestream -14 [h264 @ 0x5587f503aa00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f503aa00] error while decoding MB 0 26, bytestream 60127 [h264 @ 0x5587f4f3c9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f4f3c9c0] error while decoding MB 0 36, bytestream 111096 [h264 @ 0x5587f5a26f80] error while decoding MB 16 34, bytestream -5 [h264 @ 0x5587f5bc4c80] left block unavailable for requested i", "last_exit_at": "2026-07-06T13:27:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 297.274, "last_started_at": "2026-07-06T13:27:00+09:00", "pid": 8585, "process": {"exists": false, "pid": 8585}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 121.668, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26079, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:31:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T13:31:57+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T13:31:57+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-06T13:31:57.522967+09:00", "mtime_epoch": 1783312317.5229666, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203023}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:31:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1449.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.407, "pushed": 3.407, "read": 3.407}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.423, "count": 102, "max_ms": 5.032, "p95_ms": 4.445}, "gstreamer_push": {"avg_ms": 1.427, "count": 102, "max_ms": 3.868, "p95_ms": 1.569}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.665, "count": 102, "max_ms": 84.841, "p95_ms": 80.28}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.408, "count": 102, "max_ms": 118.968, "p95_ms": 106.555}, "yolo11_640_fire_11class": {"avg_ms": 65.586, "count": 102, "max_ms": 88.463, "p95_ms": 82.823}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.205, "count": 102, "max_ms": 97.954, "p95_ms": 92.86}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.892, "count": 102, "max_ms": 131.586, "p95_ms": 118.671}, "read_interval": {"avg_ms": 293.773, "count": 102, "max_ms": 19097.731, "p95_ms": 134.641}, "triton_wait": {"avg_ms": 81.497, "count": 102, "max_ms": 118.981, "p95_ms": 106.567}, "visualize": {"avg_ms": 3.757, "count": 102, "max_ms": 5.171, "p95_ms": 4.264}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5379.355, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:31:57+09:00", "last_error_summary": "[h264 @ 0x5587f5c93b00] error while decoding MB 77 43, bytestream -14 [h264 @ 0x5587f503aa00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f503aa00] error while decoding MB 0 26, bytestream 60127 [h264 @ 0x5587f4f3c9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f4f3c9c0] error while decoding MB 0 36, bytestream 111096 [h264 @ 0x5587f5a26f80] error while decoding MB 16 34, bytestream -5 [h264 @ 0x5587f5bc4c80] left block unavailable for requested i", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:31:57+09:00", "last_error_summary": "[h264 @ 0x5587f5c93b00] error while decoding MB 77 43, bytestream -14 [h264 @ 0x5587f503aa00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f503aa00] error while decoding MB 0 26, bytestream 60127 [h264 @ 0x5587f4f3c9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f4f3c9c0] error while decoding MB 0 36, bytestream 111096 [h264 @ 0x5587f5a26f80] error while decoding MB 16 34, bytestream -5 [h264 @ 0x5587f5bc4c80] left block unavailable for requested i", "last_exit_at": "2026-07-06T13:31:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 297.333, "last_started_at": "2026-07-06T13:27:00+09:00", "pid": 8585, "process": {"exists": false, "pid": 8585}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 121.668, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26080, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:31:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T13:31:57+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T13:31:57+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T13:31:57.522967+09:00", "mtime_epoch": 1783312317.5229666, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203023}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:31:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1449.117, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.4, "pushed": 3.434, "read": 3.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.423, "count": 102, "max_ms": 5.032, "p95_ms": 4.445}, "gstreamer_push": {"avg_ms": 1.427, "count": 103, "max_ms": 3.868, "p95_ms": 1.569}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.894, "count": 102, "max_ms": 84.841, "p95_ms": 80.28}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.675, "count": 102, "max_ms": 118.968, "p95_ms": 106.555}, "yolo11_640_fire_11class": {"avg_ms": 65.765, "count": 102, "max_ms": 88.463, "p95_ms": 82.823}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.384, "count": 102, "max_ms": 97.954, "p95_ms": 92.86}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.157, "count": 102, "max_ms": 131.586, "p95_ms": 118.671}, "read_interval": {"avg_ms": 293.773, "count": 102, "max_ms": 19097.731, "p95_ms": 134.641}, "triton_wait": {"avg_ms": 81.764, "count": 102, "max_ms": 118.981, "p95_ms": 106.567}, "visualize": {"avg_ms": 3.753, "count": 102, "max_ms": 5.171, "p95_ms": 4.264}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5379.441, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:31:57+09:00", "last_error_summary": "[h264 @ 0x5587f5c93b00] error while decoding MB 77 43, bytestream -14 [h264 @ 0x5587f503aa00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f503aa00] error while decoding MB 0 26, bytestream 60127 [h264 @ 0x5587f4f3c9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f4f3c9c0] error while decoding MB 0 36, bytestream 111096 [h264 @ 0x5587f5a26f80] error while decoding MB 16 34, bytestream -5 [h264 @ 0x5587f5bc4c80] left block unavailable for requested i", "last_exit_at": "2026-07-06T13:31:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 297.341, "last_started_at": "2026-07-06T13:27:00+09:00", "pid": 8585, "process": {"exists": false, "pid": 8585}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 121.668, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26080, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:31:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T13:31:57+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-06T13:31:57+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T13:31:57.522967+09:00", "mtime_epoch": 1783312317.5229666, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203023}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:31:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1449.117, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.401, "pushed": 3.401, "read": 3.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.423, "count": 102, "max_ms": 5.032, "p95_ms": 4.445}, "gstreamer_push": {"avg_ms": 1.428, "count": 102, "max_ms": 3.868, "p95_ms": 1.569}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.894, "count": 102, "max_ms": 84.841, "p95_ms": 80.28}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.675, "count": 102, "max_ms": 118.968, "p95_ms": 106.555}, "yolo11_640_fire_11class": {"avg_ms": 65.765, "count": 102, "max_ms": 88.463, "p95_ms": 82.823}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.384, "count": 102, "max_ms": 97.954, "p95_ms": 92.86}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.157, "count": 102, "max_ms": 131.586, "p95_ms": 118.671}, "read_interval": {"avg_ms": 293.773, "count": 102, "max_ms": 19097.731, "p95_ms": 134.641}, "triton_wait": {"avg_ms": 81.764, "count": 102, "max_ms": 118.981, "p95_ms": 106.567}, "visualize": {"avg_ms": 3.753, "count": 102, "max_ms": 5.171, "p95_ms": 4.264}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5495.648, "limit_mb": 8796093022207.996}, "detail": {"pid": 8996, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:31:57+09:00", "last_error_summary": "[h264 @ 0x5587f5c93b00] error while decoding MB 77 43, bytestream -14 [h264 @ 0x5587f503aa00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f503aa00] error while decoding MB 0 26, bytestream 60127 [h264 @ 0x5587f4f3c9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5587f4f3c9c0] error while decoding MB 0 36, bytestream 111096 [h264 @ 0x5587f5a26f80] error while decoding MB 16 34, bytestream -5 [h264 @ 0x5587f5bc4c80] left block unavailable for requested i", "last_exit_at": "2026-07-06T13:32:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:32:00+09:00", "pid": 8996, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.891, "name": "ffmpeg", "pid": 8996, "rss_mb": 141.891, "state": "S (sleeping)", "threads": 106}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 121.668, "sample_count": 2563, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26080, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:31:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.535, "last_frame_pushed_at": "2026-07-06T13:31:57+09:00", "last_frame_read_age_sec": 4.644, "last_frame_read_at": "2026-07-06T13:31:57+09:00", "mjpeg": {"age_sec": 4.516, "exists": true, "mtime": "2026-07-06T13:31:57.646969+09:00", "mtime_epoch": 1783312317.6469688, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197103}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:32:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1449.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.041, "pushed": 2.041, "read": 2.007}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.361, "count": 61, "max_ms": 4.6, "p95_ms": 4.362}, "gstreamer_push": {"avg_ms": 1.4, "count": 61, "max_ms": 3.737, "p95_ms": 1.551}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.671, "count": 61, "max_ms": 84.841, "p95_ms": 80.28}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.679, "count": 61, "max_ms": 118.968, "p95_ms": 116.385}, "yolo11_640_fire_11class": {"avg_ms": 65.527, "count": 61, "max_ms": 88.463, "p95_ms": 82.21}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.64, "count": 61, "max_ms": 97.954, "p95_ms": 92.594}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.068, "count": 61, "max_ms": 131.586, "p95_ms": 129.746}, "read_interval": {"avg_ms": 423.012, "count": 60, "max_ms": 19097.731, "p95_ms": 143.981}, "triton_wait": {"avg_ms": 80.819, "count": 61, "max_ms": 118.981, "p95_ms": 116.398}, "visualize": {"avg_ms": 3.705, "count": 61, "max_ms": 5.171, "p95_ms": 4.157}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5337.117, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a0b8973380] error while decoding MB 3 56, bytestream -15 [in#0/rtsp @ 0x55a0b7ed6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a0b7ed6e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:34:55+09:00", "last_error_summary": "[h264 @ 0x55a0b8973380] error while decoding MB 3 56, bytestream -15 [in#0/rtsp @ 0x55a0b7ed6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a0b7ed6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:31:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 214.625, "last_started_at": "2026-07-06T13:31:20+09:00", "pid": 8844, "process": {"exists": false, "pid": 8844}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.657, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23632, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:34:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.095, "last_frame_pushed_at": "2026-07-06T13:34:55+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-06T13:34:55+09:00", "mjpeg": {"age_sec": 0.076, "exists": true, "mtime": "2026-07-06T13:34:55.278272+09:00", "mtime_epoch": 1783312495.2782724, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:34:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1315.441, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.401, "pushed": 1.401, "read": 1.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.481, "count": 42, "max_ms": 4.933, "p95_ms": 4.842}, "gstreamer_push": {"avg_ms": 1.709, "count": 42, "max_ms": 6.341, "p95_ms": 4.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.464, "count": 42, "max_ms": 84.603, "p95_ms": 83.256}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.182, "count": 42, "max_ms": 124.992, "p95_ms": 112.557}, "yolo11_640_fire_11class": {"avg_ms": 63.542, "count": 42, "max_ms": 85.115, "p95_ms": 83.982}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.231, "count": 42, "max_ms": 102.525, "p95_ms": 97.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.463, "count": 42, "max_ms": 137.657, "p95_ms": 124.731}, "read_interval": {"avg_ms": 713.556, "count": 42, "max_ms": 25677.304, "p95_ms": 147.621}, "triton_wait": {"avg_ms": 77.903, "count": 42, "max_ms": 125.006, "p95_ms": 112.57}, "visualize": {"avg_ms": 3.808, "count": 42, "max_ms": 4.908, "p95_ms": 4.424}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5336.16, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:34:55+09:00", "last_error_summary": "[h264 @ 0x55a0b8973380] error while decoding MB 3 56, bytestream -15 [in#0/rtsp @ 0x55a0b7ed6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a0b7ed6e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:34:55+09:00", "last_error_summary": "[h264 @ 0x55a0b8973380] error while decoding MB 3 56, bytestream -15 [in#0/rtsp @ 0x55a0b7ed6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a0b7ed6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:34:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 214.657, "last_started_at": "2026-07-06T13:31:20+09:00", "pid": 8844, "process": {"exists": false, "pid": 8844}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.657, "sample_count": 2103, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23633, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:34:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T13:34:55+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-06T13:34:55+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T13:34:55.278272+09:00", "mtime_epoch": 1783312495.2782724, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:34:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1315.441, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.4, "pushed": 1.4, "read": 1.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.481, "count": 42, "max_ms": 4.933, "p95_ms": 4.842}, "gstreamer_push": {"avg_ms": 1.707, "count": 42, "max_ms": 6.341, "p95_ms": 4.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.79, "count": 42, "max_ms": 84.603, "p95_ms": 83.256}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.001, "count": 42, "max_ms": 124.992, "p95_ms": 112.557}, "yolo11_640_fire_11class": {"avg_ms": 64.077, "count": 42, "max_ms": 85.115, "p95_ms": 83.982}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.917, "count": 42, "max_ms": 102.525, "p95_ms": 97.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.265, "count": 42, "max_ms": 137.657, "p95_ms": 124.731}, "read_interval": {"avg_ms": 713.556, "count": 42, "max_ms": 25677.304, "p95_ms": 147.621}, "triton_wait": {"avg_ms": 78.707, "count": 42, "max_ms": 125.006, "p95_ms": 112.57}, "visualize": {"avg_ms": 3.809, "count": 42, "max_ms": 4.908, "p95_ms": 4.424}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5336.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:34:55+09:00", "last_error_summary": "[h264 @ 0x55a0b8973380] error while decoding MB 3 56, bytestream -15 [in#0/rtsp @ 0x55a0b7ed6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a0b7ed6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:34:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 214.664, "last_started_at": "2026-07-06T13:31:20+09:00", "pid": 8844, "process": {"exists": false, "pid": 8844}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.657, "sample_count": 2103, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23633, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:34:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T13:34:55+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T13:34:55+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T13:34:55.278272+09:00", "mtime_epoch": 1783312495.2782724, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:34:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1315.441, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.4, "pushed": 1.4, "read": 1.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.481, "count": 42, "max_ms": 4.933, "p95_ms": 4.842}, "gstreamer_push": {"avg_ms": 1.707, "count": 42, "max_ms": 6.341, "p95_ms": 4.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.79, "count": 42, "max_ms": 84.603, "p95_ms": 83.256}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.001, "count": 42, "max_ms": 124.992, "p95_ms": 112.557}, "yolo11_640_fire_11class": {"avg_ms": 64.077, "count": 42, "max_ms": 85.115, "p95_ms": 83.982}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.917, "count": 42, "max_ms": 102.525, "p95_ms": 97.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.265, "count": 42, "max_ms": 137.657, "p95_ms": 124.731}, "read_interval": {"avg_ms": 713.556, "count": 42, "max_ms": 25677.304, "p95_ms": 147.621}, "triton_wait": {"avg_ms": 78.707, "count": 42, "max_ms": 125.006, "p95_ms": 112.57}, "visualize": {"avg_ms": 3.809, "count": 42, "max_ms": 4.908, "p95_ms": 4.424}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5507.254, "limit_mb": 8796093022207.996}, "detail": {"pid": 9103, "restart_count": 12, "start_count": 13}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:34:55+09:00", "last_error_summary": "[h264 @ 0x55a0b8973380] error while decoding MB 3 56, bytestream -15 [in#0/rtsp @ 0x55a0b7ed6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a0b7ed6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T13:34:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:34:58+09:00", "pid": 9103, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.531, "name": "ffmpeg", "pid": 9103, "rss_mb": 206.531, "state": "S (sleeping)", "threads": 106}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.225, "sample_count": 2062, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23633, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:34:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.821, "last_frame_pushed_at": "2026-07-06T13:34:55+09:00", "last_frame_read_age_sec": 4.933, "last_frame_read_at": "2026-07-06T13:34:55+09:00", "mjpeg": {"age_sec": 4.803, "exists": true, "mtime": "2026-07-06T13:34:55.398275+09:00", "mtime_epoch": 1783312495.3982747, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226764}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:35:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1323.5, "name": "celery", "pid": 267, "rss_mb": 1315.441, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.042, "pushed": 2.042, "read": 2.042}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.313, "count": 13, "max_ms": 3.553, "p95_ms": 3.553}, "gstreamer_push": {"avg_ms": 1.927, "count": 13, "max_ms": 5.727, "p95_ms": 5.727}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.497, "count": 13, "max_ms": 84.603, "p95_ms": 84.603}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 93.333, "count": 13, "max_ms": 124.992, "p95_ms": 124.992}, "yolo11_640_fire_11class": {"avg_ms": 73.939, "count": 13, "max_ms": 85.115, "p95_ms": 85.115}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.735, "count": 13, "max_ms": 97.871, "p95_ms": 97.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.727, "count": 13, "max_ms": 137.657, "p95_ms": 137.657}, "read_interval": {"avg_ms": 2085.43, "count": 13, "max_ms": 25677.304, "p95_ms": 25677.304}, "triton_wait": {"avg_ms": 93.345, "count": 13, "max_ms": 125.006, "p95_ms": 125.006}, "visualize": {"avg_ms": 3.708, "count": 13, "max_ms": 3.829, "p95_ms": 3.829}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 6.366, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 252}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5403.594, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561182d2a2c0] cabac decode of qscale diff failed at 39 30 [h264 @ 0x561182d2a2c0] error while decoding MB 39 30, bytestream 5421 [h264 @ 0x56118378c140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56118378c140] error while decoding MB 0 6, bytestream 8423 [h264 @ 0x561182d66740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561182d66740] error while decoding MB 0 43, bytestream 3236 [h264 @ 0x561182c62440] left block unavailable for requested intra", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:40:10+09:00", "last_error_summary": "[h264 @ 0x561182d2a2c0] cabac decode of qscale diff failed at 39 30 [h264 @ 0x561182d2a2c0] error while decoding MB 39 30, bytestream 5421 [h264 @ 0x56118378c140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56118378c140] error while decoding MB 0 6, bytestream 8423 [h264 @ 0x561182d66740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561182d66740] error while decoding MB 0 43, bytestream 3236 [h264 @ 0x561182c62440] left block unavailable for requested intra", "last_exit_at": "2026-07-06T13:32:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 489.935, "last_started_at": "2026-07-06T13:32:00+09:00", "pid": 8996, "process": {"exists": false, "pid": 8996}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.03, "sample_count": 2524, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30346, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:40:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T13:40:10+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T13:40:10+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-06T13:40:10.040172+09:00", "mtime_epoch": 1783312810.0401716, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195959}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:40:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1453.758, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 106, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.535, "pushed": 3.535, "read": 3.535}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.348, "count": 106, "max_ms": 5.346, "p95_ms": 4.503}, "gstreamer_push": {"avg_ms": 1.478, "count": 106, "max_ms": 4.431, "p95_ms": 3.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.625, "count": 106, "max_ms": 81.918, "p95_ms": 75.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.295, "count": 106, "max_ms": 121.648, "p95_ms": 106.642}, "yolo11_640_fire_11class": {"avg_ms": 61.187, "count": 106, "max_ms": 85.279, "p95_ms": 83.233}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.936, "count": 106, "max_ms": 100.144, "p95_ms": 91.906}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.425, "count": 106, "max_ms": 133.714, "p95_ms": 116.28}, "read_interval": {"avg_ms": 283.256, "count": 106, "max_ms": 10124.424, "p95_ms": 136.832}, "triton_wait": {"avg_ms": 75.571, "count": 106, "max_ms": 121.661, "p95_ms": 106.654}, "visualize": {"avg_ms": 3.562, "count": 106, "max_ms": 4.494, "p95_ms": 4.306}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 252}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5403.129, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:40:10+09:00", "last_error_summary": "[h264 @ 0x561182d2a2c0] cabac decode of qscale diff failed at 39 30 [h264 @ 0x561182d2a2c0] error while decoding MB 39 30, bytestream 5421 [h264 @ 0x56118378c140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56118378c140] error while decoding MB 0 6, bytestream 8423 [h264 @ 0x561182d66740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561182d66740] error while decoding MB 0 43, bytestream 3236 [h264 @ 0x561182c62440] left block unavailable for requested intra", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:40:10+09:00", "last_error_summary": "[h264 @ 0x561182d2a2c0] cabac decode of qscale diff failed at 39 30 [h264 @ 0x561182d2a2c0] error while decoding MB 39 30, bytestream 5421 [h264 @ 0x56118378c140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56118378c140] error while decoding MB 0 6, bytestream 8423 [h264 @ 0x561182d66740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561182d66740] error while decoding MB 0 43, bytestream 3236 [h264 @ 0x561182c62440] left block unavailable for requested intra", "last_exit_at": "2026-07-06T13:40:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 489.983, "last_started_at": "2026-07-06T13:32:00+09:00", "pid": 8996, "process": {"exists": false, "pid": 8996}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.03, "sample_count": 2525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30347, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:40:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T13:40:10+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T13:40:10+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-06T13:40:10.040172+09:00", "mtime_epoch": 1783312810.0401716, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195959}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:40:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1453.758, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.542, "pushed": 3.542, "read": 3.508}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.348, "count": 106, "max_ms": 5.346, "p95_ms": 4.503}, "gstreamer_push": {"avg_ms": 1.478, "count": 106, "max_ms": 4.431, "p95_ms": 3.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.733, "count": 106, "max_ms": 81.918, "p95_ms": 75.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.263, "count": 106, "max_ms": 121.648, "p95_ms": 106.642}, "yolo11_640_fire_11class": {"avg_ms": 61.295, "count": 106, "max_ms": 85.279, "p95_ms": 83.233}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.135, "count": 106, "max_ms": 100.144, "p95_ms": 91.906}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.387, "count": 106, "max_ms": 133.714, "p95_ms": 116.28}, "read_interval": {"avg_ms": 284.84, "count": 105, "max_ms": 10124.424, "p95_ms": 136.832}, "triton_wait": {"avg_ms": 75.539, "count": 106, "max_ms": 121.661, "p95_ms": 106.654}, "visualize": {"avg_ms": 3.558, "count": 106, "max_ms": 4.494, "p95_ms": 4.306}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 252}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5402.945, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:40:10+09:00", "last_error_summary": "[h264 @ 0x561182d2a2c0] cabac decode of qscale diff failed at 39 30 [h264 @ 0x561182d2a2c0] error while decoding MB 39 30, bytestream 5421 [h264 @ 0x56118378c140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56118378c140] error while decoding MB 0 6, bytestream 8423 [h264 @ 0x561182d66740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561182d66740] error while decoding MB 0 43, bytestream 3236 [h264 @ 0x561182c62440] left block unavailable for requested intra", "last_exit_at": "2026-07-06T13:40:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 489.991, "last_started_at": "2026-07-06T13:32:00+09:00", "pid": 8996, "process": {"exists": false, "pid": 8996}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.03, "sample_count": 2525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30347, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:40:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T13:40:10+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T13:40:10+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T13:40:10.040172+09:00", "mtime_epoch": 1783312810.0401716, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195959}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:40:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1453.758, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.541, "pushed": 3.541, "read": 3.507}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.348, "count": 106, "max_ms": 5.346, "p95_ms": 4.503}, "gstreamer_push": {"avg_ms": 1.478, "count": 106, "max_ms": 4.431, "p95_ms": 3.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.733, "count": 106, "max_ms": 81.918, "p95_ms": 75.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.263, "count": 106, "max_ms": 121.648, "p95_ms": 106.642}, "yolo11_640_fire_11class": {"avg_ms": 61.295, "count": 106, "max_ms": 85.279, "p95_ms": 83.233}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.135, "count": 106, "max_ms": 100.144, "p95_ms": 91.906}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.387, "count": 106, "max_ms": 133.714, "p95_ms": 116.28}, "read_interval": {"avg_ms": 284.84, "count": 105, "max_ms": 10124.424, "p95_ms": 136.832}, "triton_wait": {"avg_ms": 75.539, "count": 106, "max_ms": 121.661, "p95_ms": 106.654}, "visualize": {"avg_ms": 3.558, "count": 106, "max_ms": 4.494, "p95_ms": 4.306}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 252}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5503.656, "limit_mb": 8796093022207.996}, "detail": {"pid": 9255, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:40:10+09:00", "last_error_summary": "[h264 @ 0x561182d2a2c0] cabac decode of qscale diff failed at 39 30 [h264 @ 0x561182d2a2c0] error while decoding MB 39 30, bytestream 5421 [h264 @ 0x56118378c140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56118378c140] error while decoding MB 0 6, bytestream 8423 [h264 @ 0x561182d66740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561182d66740] error while decoding MB 0 43, bytestream 3236 [h264 @ 0x561182c62440] left block unavailable for requested intra", "last_exit_at": "2026-07-06T13:40:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:40:12+09:00", "pid": 9255, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.547, "name": "ffmpeg", "pid": 9255, "rss_mb": 138.547, "state": "S (sleeping)", "threads": 106}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.999, "sample_count": 2487, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30347, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:40:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.358, "last_frame_pushed_at": "2026-07-06T13:40:10+09:00", "last_frame_read_age_sec": 4.461, "last_frame_read_at": "2026-07-06T13:40:10+09:00", "mjpeg": {"age_sec": 4.345, "exists": true, "mtime": "2026-07-06T13:40:10.152174+09:00", "mtime_epoch": 1783312810.1521735, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196624}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:40:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1453.758, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.136, "pushed": 2.136, "read": 2.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.342, "count": 64, "max_ms": 4.766, "p95_ms": 3.983}, "gstreamer_push": {"avg_ms": 1.574, "count": 64, "max_ms": 4.383, "p95_ms": 3.572}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.3, "count": 64, "max_ms": 81.918, "p95_ms": 75.806}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.25, "count": 64, "max_ms": 121.648, "p95_ms": 106.642}, "yolo11_640_fire_11class": {"avg_ms": 59.902, "count": 64, "max_ms": 85.279, "p95_ms": 78.26}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.909, "count": 64, "max_ms": 98.975, "p95_ms": 87.766}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.286, "count": 64, "max_ms": 133.714, "p95_ms": 116.28}, "read_interval": {"avg_ms": 406.604, "count": 63, "max_ms": 10124.424, "p95_ms": 136.937}, "triton_wait": {"avg_ms": 74.262, "count": 64, "max_ms": 121.661, "p95_ms": 106.654}, "visualize": {"avg_ms": 3.644, "count": 64, "max_ms": 4.457, "p95_ms": 4.304}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 233}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5498.984, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55823eb25000] error while decoding MB 49 34, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:44:06+09:00", "last_error_summary": "[h264 @ 0x55823eb25000] error while decoding MB 49 34, bytestream -7", "last_exit_at": "2026-07-06T13:34:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 548.131, "last_started_at": "2026-07-06T13:34:58+09:00", "pid": 9103, "process": {"exists": false, "pid": 9103}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.038, "sample_count": 2297, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28060, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:44:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.083, "last_frame_pushed_at": "2026-07-06T13:44:06+09:00", "last_frame_read_age_sec": 0.073, "last_frame_read_at": "2026-07-06T13:44:06+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-06T13:44:06.264623+09:00", "mtime_epoch": 1783313046.2646227, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 208102}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:44:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1576.727, "name": "celery", "pid": 267, "rss_mb": 1468.535, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 254, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.467, "pushed": 8.467, "read": 8.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.486, "count": 253, "max_ms": 5.795, "p95_ms": 4.033}, "gstreamer_push": {"avg_ms": 1.36, "count": 254, "max_ms": 2.171, "p95_ms": 1.49}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.223, "count": 253, "max_ms": 95.063, "p95_ms": 82.303}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 92.162, "count": 253, "max_ms": 122.772, "p95_ms": 106.061}, "yolo11_640_fire_11class": {"avg_ms": 75.205, "count": 253, "max_ms": 94.537, "p95_ms": 84.588}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 87.682, "count": 253, "max_ms": 106.882, "p95_ms": 98.973}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.985, "count": 254, "max_ms": 146.996, "p95_ms": 118.471}, "read_interval": {"avg_ms": 118.215, "count": 254, "max_ms": 158.14, "p95_ms": 132.222}, "triton_wait": {"avg_ms": 92.478, "count": 253, "max_ms": 122.785, "p95_ms": 106.074}, "visualize": {"avg_ms": 3.735, "count": 253, "max_ms": 5.839, "p95_ms": 4.385}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 233}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5498.367, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:44:06+09:00", "last_error_summary": "[h264 @ 0x55823eb25000] error while decoding MB 49 34, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:44:06+09:00", "last_error_summary": "[h264 @ 0x55823eb25000] error while decoding MB 49 34, bytestream -7", "last_exit_at": "2026-07-06T13:44:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 548.163, "last_started_at": "2026-07-06T13:34:58+09:00", "pid": 9103, "process": {"exists": false, "pid": 9103}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.038, "sample_count": 2298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28061, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:44:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T13:44:06+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-06T13:44:06+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-06T13:44:06.264623+09:00", "mtime_epoch": 1783313046.2646227, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 208102}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:44:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1576.727, "name": "celery", "pid": 267, "rss_mb": 1468.535, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.492, "pushed": 8.492, "read": 8.458}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.485, "count": 254, "max_ms": 5.795, "p95_ms": 4.033}, "gstreamer_push": {"avg_ms": 1.361, "count": 254, "max_ms": 2.171, "p95_ms": 1.49}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.199, "count": 254, "max_ms": 95.063, "p95_ms": 82.303}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 92.132, "count": 254, "max_ms": 122.772, "p95_ms": 106.061}, "yolo11_640_fire_11class": {"avg_ms": 75.186, "count": 254, "max_ms": 94.537, "p95_ms": 84.588}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 87.641, "count": 254, "max_ms": 106.882, "p95_ms": 98.973}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.967, "count": 254, "max_ms": 146.996, "p95_ms": 118.471}, "read_interval": {"avg_ms": 118.243, "count": 253, "max_ms": 158.14, "p95_ms": 132.222}, "triton_wait": {"avg_ms": 92.447, "count": 254, "max_ms": 122.785, "p95_ms": 106.074}, "visualize": {"avg_ms": 3.735, "count": 254, "max_ms": 5.839, "p95_ms": 4.385}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 233}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5497.715, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:44:06+09:00", "last_error_summary": "[h264 @ 0x55823eb25000] error while decoding MB 49 34, bytestream -7", "last_exit_at": "2026-07-06T13:44:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 548.172, "last_started_at": "2026-07-06T13:34:58+09:00", "pid": 9103, "process": {"exists": false, "pid": 9103}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.038, "sample_count": 2298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28061, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:44:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T13:44:06+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-06T13:44:06+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T13:44:06.264623+09:00", "mtime_epoch": 1783313046.2646227, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 208102}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:44:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1576.727, "name": "celery", "pid": 267, "rss_mb": 1468.535, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.489, "pushed": 8.489, "read": 8.456}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.485, "count": 254, "max_ms": 5.795, "p95_ms": 4.033}, "gstreamer_push": {"avg_ms": 1.361, "count": 254, "max_ms": 2.171, "p95_ms": 1.49}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.199, "count": 254, "max_ms": 95.063, "p95_ms": 82.303}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 92.132, "count": 254, "max_ms": 122.772, "p95_ms": 106.061}, "yolo11_640_fire_11class": {"avg_ms": 75.186, "count": 254, "max_ms": 94.537, "p95_ms": 84.588}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 87.641, "count": 254, "max_ms": 106.882, "p95_ms": 98.973}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.967, "count": 254, "max_ms": 146.996, "p95_ms": 118.471}, "read_interval": {"avg_ms": 118.243, "count": 253, "max_ms": 158.14, "p95_ms": 132.222}, "triton_wait": {"avg_ms": 92.447, "count": 254, "max_ms": 122.785, "p95_ms": 106.074}, "visualize": {"avg_ms": 3.735, "count": 254, "max_ms": 5.839, "p95_ms": 4.385}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 233}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5609.098, "limit_mb": 8796093022207.996}, "detail": {"pid": 9423, "restart_count": 13, "start_count": 14}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:44:06+09:00", "last_error_summary": "[h264 @ 0x55823eb25000] error while decoding MB 49 34, bytestream -7", "last_exit_at": "2026-07-06T13:44:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:44:08+09:00", "pid": 9423, "process": {"exists": true, "fd_count": 5, "hwm_mb": 148.391, "name": "ffmpeg", "pid": 9423, "rss_mb": 148.391, "state": "R (running)", "threads": 49}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.341, "sample_count": 2257, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28061, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:44:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.471, "last_frame_pushed_at": "2026-07-06T13:44:06+09:00", "last_frame_read_age_sec": 4.57, "last_frame_read_at": "2026-07-06T13:44:06+09:00", "mjpeg": {"age_sec": 4.455, "exists": true, "mtime": "2026-07-06T13:44:06.372625+09:00", "mtime_epoch": 1783313046.3726246, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 204608}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:44:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1576.727, "name": "celery", "pid": 267, "rss_mb": 1468.535, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.168, "pushed": 7.168, "read": 7.168}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.505, "count": 215, "max_ms": 5.795, "p95_ms": 4.537}, "gstreamer_push": {"avg_ms": 1.365, "count": 215, "max_ms": 2.171, "p95_ms": 1.492}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.54, "count": 215, "max_ms": 95.063, "p95_ms": 82.308}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 92.58, "count": 215, "max_ms": 122.772, "p95_ms": 106.082}, "yolo11_640_fire_11class": {"avg_ms": 75.401, "count": 215, "max_ms": 94.537, "p95_ms": 84.732}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 87.995, "count": 215, "max_ms": 106.882, "p95_ms": 99.203}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.445, "count": 215, "max_ms": 146.996, "p95_ms": 118.529}, "read_interval": {"avg_ms": 118.784, "count": 215, "max_ms": 158.14, "p95_ms": 132.565}, "triton_wait": {"avg_ms": 92.821, "count": 215, "max_ms": 122.785, "p95_ms": 106.094}, "visualize": {"avg_ms": 3.765, "count": 215, "max_ms": 5.839, "p95_ms": 4.469}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 274}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5519.836, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5599c3601740] left block unavailable for requested intra mode [h264 @ 0x5599c3601740] error while decoding MB 0 28, bytestream 1327 [h264 @ 0x5599c2992340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5599c2992340] error while decoding MB 0 26, bytestream 6012 [h264 @ 0x5599c29a6500] cabac decode of qscale diff failed at 49 41 [h264 @ 0x5599c29a6500] error while decoding MB 49 41, bytestream 7792 [h264 @ 0x5599c290fdc0] cabac decode of qscale diff failed at 51 40 [h2", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:45:08+09:00", "last_error_summary": "[h264 @ 0x5599c3601740] left block unavailable for requested intra mode [h264 @ 0x5599c3601740] error while decoding MB 0 28, bytestream 1327 [h264 @ 0x5599c2992340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5599c2992340] error while decoding MB 0 26, bytestream 6012 [h264 @ 0x5599c29a6500] cabac decode of qscale diff failed at 49 41 [h264 @ 0x5599c29a6500] error while decoding MB 49 41, bytestream 7792 [h264 @ 0x5599c290fdc0] cabac decode of qscale diff failed at 51 40 [h2", "last_exit_at": "2026-07-06T13:40:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 295.628, "last_started_at": "2026-07-06T13:40:12+09:00", "pid": 9255, "process": {"exists": false, "pid": 9255}, "restart_count": 7, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.027, "sample_count": 2580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32916, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:45:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T13:45:08+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-06T13:45:08+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T13:45:08.073789+09:00", "mtime_epoch": 1783313108.0737894, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189826}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:45:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1455.543, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.704, "pushed": 3.704, "read": 3.704}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.337, "count": 111, "max_ms": 5.127, "p95_ms": 4.08}, "gstreamer_push": {"avg_ms": 1.769, "count": 111, "max_ms": 6.771, "p95_ms": 6.079}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.566, "count": 111, "max_ms": 79.122, "p95_ms": 72.438}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.591, "count": 111, "max_ms": 110.845, "p95_ms": 101.657}, "yolo11_640_fire_11class": {"avg_ms": 57.4, "count": 111, "max_ms": 86.067, "p95_ms": 74.826}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.042, "count": 111, "max_ms": 97.249, "p95_ms": 90.324}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.729, "count": 111, "max_ms": 123.763, "p95_ms": 114.606}, "read_interval": {"avg_ms": 270.133, "count": 111, "max_ms": 19286.784, "p95_ms": 130.471}, "triton_wait": {"avg_ms": 71.666, "count": 111, "max_ms": 110.858, "p95_ms": 101.67}, "visualize": {"avg_ms": 3.64, "count": 111, "max_ms": 5.102, "p95_ms": 4.245}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 274}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5519.664, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:45:08+09:00", "last_error_summary": "[h264 @ 0x5599c3601740] left block unavailable for requested intra mode [h264 @ 0x5599c3601740] error while decoding MB 0 28, bytestream 1327 [h264 @ 0x5599c2992340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5599c2992340] error while decoding MB 0 26, bytestream 6012 [h264 @ 0x5599c29a6500] cabac decode of qscale diff failed at 49 41 [h264 @ 0x5599c29a6500] error while decoding MB 49 41, bytestream 7792 [h264 @ 0x5599c290fdc0] cabac decode of qscale diff failed at 51 40 [h2", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:45:08+09:00", "last_error_summary": "[h264 @ 0x5599c3601740] left block unavailable for requested intra mode [h264 @ 0x5599c3601740] error while decoding MB 0 28, bytestream 1327 [h264 @ 0x5599c2992340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5599c2992340] error while decoding MB 0 26, bytestream 6012 [h264 @ 0x5599c29a6500] cabac decode of qscale diff failed at 49 41 [h264 @ 0x5599c29a6500] error while decoding MB 49 41, bytestream 7792 [h264 @ 0x5599c290fdc0] cabac decode of qscale diff failed at 51 40 [h2", "last_exit_at": "2026-07-06T13:45:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 295.657, "last_started_at": "2026-07-06T13:40:12+09:00", "pid": 9255, "process": {"exists": false, "pid": 9255}, "restart_count": 7, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.027, "sample_count": 2581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:45:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T13:45:08+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-06T13:45:08+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-06T13:45:08.073789+09:00", "mtime_epoch": 1783313108.0737894, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189826}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:45:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1455.543, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.733, "pushed": 3.733, "read": 3.7}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.365, "count": 111, "max_ms": 6.426, "p95_ms": 4.149}, "gstreamer_push": {"avg_ms": 1.766, "count": 112, "max_ms": 6.771, "p95_ms": 6.079}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.526, "count": 111, "max_ms": 79.122, "p95_ms": 72.438}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.611, "count": 111, "max_ms": 110.845, "p95_ms": 101.657}, "yolo11_640_fire_11class": {"avg_ms": 57.419, "count": 111, "max_ms": 86.067, "p95_ms": 74.826}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.046, "count": 111, "max_ms": 97.249, "p95_ms": 90.324}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.704, "count": 112, "max_ms": 123.763, "p95_ms": 114.606}, "read_interval": {"avg_ms": 270.133, "count": 111, "max_ms": 19286.784, "p95_ms": 130.471}, "triton_wait": {"avg_ms": 71.686, "count": 111, "max_ms": 110.858, "p95_ms": 101.67}, "visualize": {"avg_ms": 3.648, "count": 111, "max_ms": 5.102, "p95_ms": 4.255}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 274}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5519.719, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:45:08+09:00", "last_error_summary": "[h264 @ 0x5599c3601740] left block unavailable for requested intra mode [h264 @ 0x5599c3601740] error while decoding MB 0 28, bytestream 1327 [h264 @ 0x5599c2992340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5599c2992340] error while decoding MB 0 26, bytestream 6012 [h264 @ 0x5599c29a6500] cabac decode of qscale diff failed at 49 41 [h264 @ 0x5599c29a6500] error while decoding MB 49 41, bytestream 7792 [h264 @ 0x5599c290fdc0] cabac decode of qscale diff failed at 51 40 [h2", "last_exit_at": "2026-07-06T13:45:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 295.665, "last_started_at": "2026-07-06T13:40:12+09:00", "pid": 9255, "process": {"exists": false, "pid": 9255}, "restart_count": 7, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.027, "sample_count": 2581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:45:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T13:45:08+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-06T13:45:08+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-06T13:45:08.073789+09:00", "mtime_epoch": 1783313108.0737894, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189826}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:45:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1455.543, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.701, "pushed": 3.701, "read": 3.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.365, "count": 111, "max_ms": 6.426, "p95_ms": 4.149}, "gstreamer_push": {"avg_ms": 1.77, "count": 111, "max_ms": 6.771, "p95_ms": 6.079}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.526, "count": 111, "max_ms": 79.122, "p95_ms": 72.438}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.611, "count": 111, "max_ms": 110.845, "p95_ms": 101.657}, "yolo11_640_fire_11class": {"avg_ms": 57.419, "count": 111, "max_ms": 86.067, "p95_ms": 74.826}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.046, "count": 111, "max_ms": 97.249, "p95_ms": 90.324}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.786, "count": 111, "max_ms": 123.763, "p95_ms": 114.606}, "read_interval": {"avg_ms": 270.133, "count": 111, "max_ms": 19286.784, "p95_ms": 130.471}, "triton_wait": {"avg_ms": 71.686, "count": 111, "max_ms": 110.858, "p95_ms": 101.67}, "visualize": {"avg_ms": 3.648, "count": 111, "max_ms": 5.102, "p95_ms": 4.255}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 274}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5517.242, "limit_mb": 8796093022207.996}, "detail": {"pid": 9575, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:45:08+09:00", "last_error_summary": "[h264 @ 0x5599c3601740] left block unavailable for requested intra mode [h264 @ 0x5599c3601740] error while decoding MB 0 28, bytestream 1327 [h264 @ 0x5599c2992340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5599c2992340] error while decoding MB 0 26, bytestream 6012 [h264 @ 0x5599c29a6500] cabac decode of qscale diff failed at 49 41 [h264 @ 0x5599c29a6500] error while decoding MB 49 41, bytestream 7792 [h264 @ 0x5599c290fdc0] cabac decode of qscale diff failed at 51 40 [h2", "last_exit_at": "2026-07-06T13:45:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:45:10+09:00", "pid": 9575, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.152, "name": "ffmpeg", "pid": 9575, "rss_mb": 133.152, "state": "S (sleeping)", "threads": 106}, "restart_count": 8, "restart_count_per_hour": 7, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.027, "sample_count": 2570, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:45:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.683, "last_frame_pushed_at": "2026-07-06T13:45:08+09:00", "last_frame_read_age_sec": 4.766, "last_frame_read_at": "2026-07-06T13:45:08+09:00", "mjpeg": {"age_sec": 4.669, "exists": true, "mtime": "2026-07-06T13:45:08.161791+09:00", "mtime_epoch": 1783313108.161791, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:45:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1455.543, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.169, "pushed": 2.169, "read": 2.136}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.348, "count": 65, "max_ms": 6.426, "p95_ms": 4.149}, "gstreamer_push": {"avg_ms": 1.69, "count": 65, "max_ms": 6.161, "p95_ms": 4.071}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.781, "count": 65, "max_ms": 79.122, "p95_ms": 71.568}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.085, "count": 65, "max_ms": 110.845, "p95_ms": 97.973}, "yolo11_640_fire_11class": {"avg_ms": 56.879, "count": 65, "max_ms": 79.345, "p95_ms": 72.806}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.082, "count": 65, "max_ms": 97.249, "p95_ms": 88.916}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.147, "count": 65, "max_ms": 123.067, "p95_ms": 110.476}, "read_interval": {"avg_ms": 395.325, "count": 64, "max_ms": 19286.784, "p95_ms": 128.608}, "triton_wait": {"avg_ms": 70.177, "count": 65, "max_ms": 110.858, "p95_ms": 97.991}, "visualize": {"avg_ms": 3.616, "count": 65, "max_ms": 5.102, "p95_ms": 4.255}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5415.418, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564930332680] error while decoding MB 5 42, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:54:09+09:00", "last_error_summary": "[h264 @ 0x564930332680] error while decoding MB 5 42, bytestream -7", "last_exit_at": "2026-07-06T13:45:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 538.212, "last_started_at": "2026-07-06T13:45:10+09:00", "pid": 9575, "process": {"exists": false, "pid": 9575}, "restart_count": 8, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.376, "sample_count": 2711, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37773, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:54:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-06T13:54:08+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T13:54:08+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T13:54:08.998273+09:00", "mtime_epoch": 1783313648.9982727, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190658}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:54:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1454.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.548, "pushed": 7.548, "read": 7.548}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.775, "count": 226, "max_ms": 7.583, "p95_ms": 6.381}, "gstreamer_push": {"avg_ms": 1.377, "count": 226, "max_ms": 7.171, "p95_ms": 2.249}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.861, "count": 226, "max_ms": 81.38, "p95_ms": 77.454}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.842, "count": 226, "max_ms": 115.487, "p95_ms": 103.457}, "yolo11_640_fire_11class": {"avg_ms": 66.964, "count": 226, "max_ms": 89.555, "p95_ms": 81.125}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.22, "count": 226, "max_ms": 103.891, "p95_ms": 95.017}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.068, "count": 226, "max_ms": 183.871, "p95_ms": 152.426}, "read_interval": {"avg_ms": 132.865, "count": 226, "max_ms": 194.947, "p95_ms": 163.636}, "triton_wait": {"avg_ms": 83.242, "count": 226, "max_ms": 115.499, "p95_ms": 103.471}, "visualize": {"avg_ms": 4.632, "count": 226, "max_ms": 6.678, "p95_ms": 5.564}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5417.152, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:54:09+09:00", "last_error_summary": "[h264 @ 0x564930332680] error while decoding MB 5 42, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:54:09+09:00", "last_error_summary": "[h264 @ 0x564930332680] error while decoding MB 5 42, bytestream -7", "last_exit_at": "2026-07-06T13:54:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 538.278, "last_started_at": "2026-07-06T13:45:10+09:00", "pid": 9575, "process": {"exists": false, "pid": 9575}, "restart_count": 8, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.376, "sample_count": 2712, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37774, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:54:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T13:54:09+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-06T13:54:08+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T13:54:08.998273+09:00", "mtime_epoch": 1783313648.9982727, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190658}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:54:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1454.645, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.534, "pushed": 7.534, "read": 7.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.761, "count": 226, "max_ms": 7.583, "p95_ms": 6.351}, "gstreamer_push": {"avg_ms": 1.378, "count": 226, "max_ms": 7.171, "p95_ms": 2.249}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.899, "count": 226, "max_ms": 81.38, "p95_ms": 77.454}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.894, "count": 226, "max_ms": 115.487, "p95_ms": 103.457}, "yolo11_640_fire_11class": {"avg_ms": 67.022, "count": 226, "max_ms": 89.555, "p95_ms": 81.125}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.291, "count": 226, "max_ms": 103.891, "p95_ms": 95.017}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.931, "count": 226, "max_ms": 183.871, "p95_ms": 152.426}, "read_interval": {"avg_ms": 132.865, "count": 226, "max_ms": 194.947, "p95_ms": 163.636}, "triton_wait": {"avg_ms": 83.293, "count": 226, "max_ms": 115.499, "p95_ms": 103.471}, "visualize": {"avg_ms": 4.625, "count": 226, "max_ms": 6.678, "p95_ms": 5.564}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5417.211, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:54:09+09:00", "last_error_summary": "[h264 @ 0x564930332680] error while decoding MB 5 42, bytestream -7", "last_exit_at": "2026-07-06T13:54:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 538.287, "last_started_at": "2026-07-06T13:45:10+09:00", "pid": 9575, "process": {"exists": false, "pid": 9575}, "restart_count": 8, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.376, "sample_count": 2712, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37774, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:54:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T13:54:09+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T13:54:08+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-06T13:54:08.998273+09:00", "mtime_epoch": 1783313648.9982727, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190658}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:54:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1454.645, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.568, "pushed": 7.568, "read": 7.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.761, "count": 226, "max_ms": 7.583, "p95_ms": 6.351}, "gstreamer_push": {"avg_ms": 1.378, "count": 226, "max_ms": 7.171, "p95_ms": 2.249}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.899, "count": 226, "max_ms": 81.38, "p95_ms": 77.454}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.894, "count": 226, "max_ms": 115.487, "p95_ms": 103.457}, "yolo11_640_fire_11class": {"avg_ms": 67.022, "count": 226, "max_ms": 89.555, "p95_ms": 81.125}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.291, "count": 226, "max_ms": 103.891, "p95_ms": 95.017}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.931, "count": 226, "max_ms": 183.871, "p95_ms": 152.426}, "read_interval": {"avg_ms": 132.782, "count": 225, "max_ms": 194.947, "p95_ms": 163.636}, "triton_wait": {"avg_ms": 83.293, "count": 226, "max_ms": 115.499, "p95_ms": 103.471}, "visualize": {"avg_ms": 4.625, "count": 226, "max_ms": 6.678, "p95_ms": 5.564}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5531.438, "limit_mb": 8796093022207.996}, "detail": {"pid": 9742, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:54:09+09:00", "last_error_summary": "[h264 @ 0x564930332680] error while decoding MB 5 42, bytestream -7", "last_exit_at": "2026-07-06T13:54:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T13:54:11+09:00", "pid": 9742, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.27, "name": "ffmpeg", "pid": 9742, "rss_mb": 139.27, "state": "S (sleeping)", "threads": 106}, "restart_count": 9, "restart_count_per_hour": 7, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.699, "sample_count": 2669, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37774, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:54:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.685, "last_frame_pushed_at": "2026-07-06T13:54:09+09:00", "last_frame_read_age_sec": 4.798, "last_frame_read_at": "2026-07-06T13:54:08+09:00", "mjpeg": {"age_sec": 4.667, "exists": true, "mtime": "2026-07-06T13:54:09.118276+09:00", "mtime_epoch": 1783313649.1182761, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191489}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T13:54:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1454.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 192, "pushed_frames": 192, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.404, "pushed": 6.404, "read": 6.371}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.612, "count": 191, "max_ms": 7.583, "p95_ms": 5.913}, "gstreamer_push": {"avg_ms": 1.41, "count": 192, "max_ms": 7.171, "p95_ms": 2.309}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.557, "count": 191, "max_ms": 81.38, "p95_ms": 77.733}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.053, "count": 191, "max_ms": 115.487, "p95_ms": 104.46}, "yolo11_640_fire_11class": {"avg_ms": 68.735, "count": 191, "max_ms": 89.28, "p95_ms": 81.393}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.503, "count": 191, "max_ms": 103.541, "p95_ms": 95.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.541, "count": 192, "max_ms": 165.537, "p95_ms": 152.426}, "read_interval": {"avg_ms": 132.575, "count": 191, "max_ms": 178.727, "p95_ms": 163.636}, "triton_wait": {"avg_ms": 85.523, "count": 191, "max_ms": 115.499, "p95_ms": 104.474}, "visualize": {"avg_ms": 4.547, "count": 191, "max_ms": 6.669, "p95_ms": 5.564}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 291}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5379.57, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55eacbe43d00] error while decoding MB 84 43, bytestream -19", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:59:58+09:00", "last_error_summary": "[h264 @ 0x55eacbe43d00] error while decoding MB 84 43, bytestream -19", "last_exit_at": "2026-07-06T13:44:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 949.187, "last_started_at": "2026-07-06T13:44:08+09:00", "pid": 9423, "process": {"exists": false, "pid": 9423}, "restart_count": 13, "restart_count_per_hour": 12, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 212.497, "sample_count": 1991, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34969, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:59:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T13:59:57+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T13:59:57+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T13:59:57.963363+09:00", "mtime_epoch": 1783313997.9633627, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 215108}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:59:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1322.301, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.586, "pushed": 6.586, "read": 6.586}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.866, "count": 196, "max_ms": 11.215, "p95_ms": 9.708}, "gstreamer_push": {"avg_ms": 1.297, "count": 197, "max_ms": 6.433, "p95_ms": 3.044}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.17, "count": 196, "max_ms": 90.214, "p95_ms": 78.455}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.624, "count": 196, "max_ms": 115.015, "p95_ms": 106.92}, "yolo11_640_fire_11class": {"avg_ms": 64.141, "count": 196, "max_ms": 89.786, "p95_ms": 84.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.436, "count": 196, "max_ms": 108.083, "p95_ms": 99.83}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.805, "count": 197, "max_ms": 231.423, "p95_ms": 212.438}, "read_interval": {"avg_ms": 152.411, "count": 197, "max_ms": 238.734, "p95_ms": 221.84}, "triton_wait": {"avg_ms": 80.139, "count": 196, "max_ms": 115.027, "p95_ms": 106.931}, "visualize": {"avg_ms": 5.223, "count": 196, "max_ms": 7.262, "p95_ms": 6.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 291}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5378.832, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T13:59:58+09:00", "last_error_summary": "[h264 @ 0x55eacbe43d00] error while decoding MB 84 43, bytestream -19", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:59:58+09:00", "last_error_summary": "[h264 @ 0x55eacbe43d00] error while decoding MB 84 43, bytestream -19", "last_exit_at": "2026-07-06T13:59:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 949.286, "last_started_at": "2026-07-06T13:44:08+09:00", "pid": 9423, "process": {"exists": false, "pid": 9423}, "restart_count": 13, "restart_count_per_hour": 12, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 212.497, "sample_count": 1991, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:59:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T13:59:58+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-06T13:59:57+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-06T13:59:57.963363+09:00", "mtime_epoch": 1783313997.9633627, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 215108}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:59:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1322.301, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.567, "pushed": 6.567, "read": 6.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.863, "count": 197, "max_ms": 11.215, "p95_ms": 9.708}, "gstreamer_push": {"avg_ms": 1.287, "count": 197, "max_ms": 6.433, "p95_ms": 2.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.154, "count": 197, "max_ms": 90.214, "p95_ms": 78.455}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.715, "count": 197, "max_ms": 115.015, "p95_ms": 106.92}, "yolo11_640_fire_11class": {"avg_ms": 64.228, "count": 197, "max_ms": 89.786, "p95_ms": 84.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.46, "count": 197, "max_ms": 108.083, "p95_ms": 99.83}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.69, "count": 197, "max_ms": 231.423, "p95_ms": 212.438}, "read_interval": {"avg_ms": 152.411, "count": 197, "max_ms": 238.734, "p95_ms": 221.84}, "triton_wait": {"avg_ms": 80.227, "count": 197, "max_ms": 115.027, "p95_ms": 106.931}, "visualize": {"avg_ms": 5.222, "count": 197, "max_ms": 7.262, "p95_ms": 6.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 291}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5378.801, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:59:58+09:00", "last_error_summary": "[h264 @ 0x55eacbe43d00] error while decoding MB 84 43, bytestream -19", "last_exit_at": "2026-07-06T13:59:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 949.291, "last_started_at": "2026-07-06T13:44:08+09:00", "pid": 9423, "process": {"exists": false, "pid": 9423}, "restart_count": 13, "restart_count_per_hour": 12, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 212.497, "sample_count": 1991, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:59:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T13:59:58+09:00", "last_frame_read_age_sec": 0.165, "last_frame_read_at": "2026-07-06T13:59:57+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-06T13:59:57.963363+09:00", "mtime_epoch": 1783313997.9633627, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 215108}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T13:59:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1322.301, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.604, "pushed": 6.604, "read": 6.571}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.863, "count": 197, "max_ms": 11.215, "p95_ms": 9.708}, "gstreamer_push": {"avg_ms": 1.287, "count": 197, "max_ms": 6.433, "p95_ms": 2.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.154, "count": 197, "max_ms": 90.214, "p95_ms": 78.455}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.715, "count": 197, "max_ms": 115.015, "p95_ms": 106.92}, "yolo11_640_fire_11class": {"avg_ms": 64.228, "count": 197, "max_ms": 89.786, "p95_ms": 84.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.46, "count": 197, "max_ms": 108.083, "p95_ms": 99.83}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.69, "count": 197, "max_ms": 231.423, "p95_ms": 212.438}, "read_interval": {"avg_ms": 152.239, "count": 196, "max_ms": 238.734, "p95_ms": 221.84}, "triton_wait": {"avg_ms": 80.227, "count": 197, "max_ms": 115.027, "p95_ms": 106.931}, "visualize": {"avg_ms": 5.222, "count": 197, "max_ms": 7.262, "p95_ms": 6.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 291}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5458.883, "limit_mb": 8796093022207.996}, "detail": {"pid": 9849, "restart_count": 14, "start_count": 15}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T13:59:58+09:00", "last_error_summary": "[h264 @ 0x55eacbe43d00] error while decoding MB 84 43, bytestream -19", "last_exit_at": "2026-07-06T14:00:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:00:00+09:00", "pid": 9849, "process": {"exists": true, "fd_count": 5, "hwm_mb": 108.191, "name": "ffmpeg", "pid": 9849, "rss_mb": 108.191, "state": "S (sleeping)", "threads": 19}, "restart_count": 14, "restart_count_per_hour": 13, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 212.653, "sample_count": 1951, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T13:59:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.561, "last_frame_pushed_at": "2026-07-06T13:59:58+09:00", "last_frame_read_age_sec": 4.714, "last_frame_read_at": "2026-07-06T13:59:57+09:00", "mjpeg": {"age_sec": 4.543, "exists": true, "mtime": "2026-07-06T13:59:58.123367+09:00", "mtime_epoch": 1783313998.1233666, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 215108}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:00:02+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1322.301, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.836, "pushed": 5.836, "read": 5.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.533, "count": 174, "max_ms": 11.215, "p95_ms": 9.403}, "gstreamer_push": {"avg_ms": 1.319, "count": 175, "max_ms": 6.433, "p95_ms": 2.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.664, "count": 174, "max_ms": 90.214, "p95_ms": 78.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.303, "count": 174, "max_ms": 115.015, "p95_ms": 103.759}, "yolo11_640_fire_11class": {"avg_ms": 64.367, "count": 174, "max_ms": 89.786, "p95_ms": 84.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.214, "count": 174, "max_ms": 108.083, "p95_ms": 99.634}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.61, "count": 175, "max_ms": 231.423, "p95_ms": 208.403}, "read_interval": {"avg_ms": 146.394, "count": 174, "max_ms": 238.734, "p95_ms": 216.038}, "triton_wait": {"avg_ms": 79.712, "count": 174, "max_ms": 115.027, "p95_ms": 103.775}, "visualize": {"avg_ms": 5.07, "count": 174, "max_ms": 7.242, "p95_ms": 6.635}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5430.906, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 12, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56486be82980] cabac decode of qscale diff failed at 23 8 [h264 @ 0x56486be82980] error while decoding MB 23 8, bytestream 1881 [h264 @ 0x56486bb46f80] top block unavailable for requested intra mode [h264 @ 0x56486bb46f80] error while decoding MB 45 0, bytestream 5003 [h264 @ 0x56486bdb3b00] cabac decode of qscale diff failed at 70 36 [h264 @ 0x56486bdb3b00] error while decoding MB 70 36, bytestream 7012 [h264 @ 0x56486be82980] cabac decode of qscale diff failed at 35 42 [h264 @ 0x56486", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:04:34+09:00", "last_error_summary": "[h264 @ 0x56486be82980] cabac decode of qscale diff failed at 23 8 [h264 @ 0x56486be82980] error while decoding MB 23 8, bytestream 1881 [h264 @ 0x56486bb46f80] top block unavailable for requested intra mode [h264 @ 0x56486bb46f80] error while decoding MB 45 0, bytestream 5003 [h264 @ 0x56486bdb3b00] cabac decode of qscale diff failed at 70 36 [h264 @ 0x56486bdb3b00] error while decoding MB 70 36, bytestream 7012 [h264 @ 0x56486be82980] cabac decode of qscale diff failed at 35 42 [h264 @ 0x56486", "last_exit_at": "2026-07-06T13:54:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 622.957, "last_started_at": "2026-07-06T13:54:11+09:00", "pid": 9742, "process": {"exists": false, "pid": 9742}, "restart_count": 9, "restart_count_per_hour": 6, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.741, "sample_count": 2736, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43390, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:04:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T14:04:34+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-06T14:04:34+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-06T14:04:34.693922+09:00", "mtime_epoch": 1783314274.6939216, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173952}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:04:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1460.254, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.709, "pushed": 3.709, "read": 3.709}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.308, "count": 111, "max_ms": 4.324, "p95_ms": 4.034}, "gstreamer_push": {"avg_ms": 1.51, "count": 111, "max_ms": 6.353, "p95_ms": 3.555}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.077, "count": 111, "max_ms": 84.877, "p95_ms": 74.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.966, "count": 111, "max_ms": 115.55, "p95_ms": 102.764}, "yolo11_640_fire_11class": {"avg_ms": 58.735, "count": 111, "max_ms": 86.379, "p95_ms": 80.43}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.725, "count": 111, "max_ms": 99.849, "p95_ms": 92.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.944, "count": 111, "max_ms": 129.32, "p95_ms": 112.851}, "read_interval": {"avg_ms": 269.758, "count": 111, "max_ms": 19317.486, "p95_ms": 130.673}, "triton_wait": {"avg_ms": 71.028, "count": 111, "max_ms": 115.564, "p95_ms": 102.78}, "visualize": {"avg_ms": 3.632, "count": 111, "max_ms": 4.571, "p95_ms": 4.21}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5430.277, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:04:34+09:00", "last_error_summary": "[h264 @ 0x56486be82980] cabac decode of qscale diff failed at 23 8 [h264 @ 0x56486be82980] error while decoding MB 23 8, bytestream 1881 [h264 @ 0x56486bb46f80] top block unavailable for requested intra mode [h264 @ 0x56486bb46f80] error while decoding MB 45 0, bytestream 5003 [h264 @ 0x56486bdb3b00] cabac decode of qscale diff failed at 70 36 [h264 @ 0x56486bdb3b00] error while decoding MB 70 36, bytestream 7012 [h264 @ 0x56486be82980] cabac decode of qscale diff failed at 35 42 [h264 @ 0x56486", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:04:34+09:00", "last_error_summary": "[h264 @ 0x56486be82980] cabac decode of qscale diff failed at 23 8 [h264 @ 0x56486be82980] error while decoding MB 23 8, bytestream 1881 [h264 @ 0x56486bb46f80] top block unavailable for requested intra mode [h264 @ 0x56486bb46f80] error while decoding MB 45 0, bytestream 5003 [h264 @ 0x56486bdb3b00] cabac decode of qscale diff failed at 70 36 [h264 @ 0x56486bdb3b00] error while decoding MB 70 36, bytestream 7012 [h264 @ 0x56486be82980] cabac decode of qscale diff failed at 35 42 [h264 @ 0x56486", "last_exit_at": "2026-07-06T14:04:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 622.993, "last_started_at": "2026-07-06T13:54:11+09:00", "pid": 9742, "process": {"exists": false, "pid": 9742}, "restart_count": 9, "restart_count_per_hour": 6, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.741, "sample_count": 2737, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:04:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T14:04:34+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-06T14:04:34+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-06T14:04:34.693922+09:00", "mtime_epoch": 1783314274.6939216, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173952}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:04:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1460.254, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.738, "pushed": 3.738, "read": 3.705}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.349, "count": 112, "max_ms": 7.87, "p95_ms": 4.05}, "gstreamer_push": {"avg_ms": 1.508, "count": 112, "max_ms": 6.353, "p95_ms": 3.555}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.04, "count": 112, "max_ms": 84.877, "p95_ms": 74.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.948, "count": 112, "max_ms": 115.55, "p95_ms": 102.764}, "yolo11_640_fire_11class": {"avg_ms": 58.733, "count": 112, "max_ms": 86.379, "p95_ms": 80.43}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.698, "count": 112, "max_ms": 99.849, "p95_ms": 92.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.972, "count": 112, "max_ms": 129.32, "p95_ms": 112.851}, "read_interval": {"avg_ms": 269.758, "count": 111, "max_ms": 19317.486, "p95_ms": 130.673}, "triton_wait": {"avg_ms": 71.009, "count": 112, "max_ms": 115.564, "p95_ms": 102.78}, "visualize": {"avg_ms": 3.638, "count": 112, "max_ms": 4.571, "p95_ms": 4.245}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5430.27, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:04:34+09:00", "last_error_summary": "[h264 @ 0x56486be82980] cabac decode of qscale diff failed at 23 8 [h264 @ 0x56486be82980] error while decoding MB 23 8, bytestream 1881 [h264 @ 0x56486bb46f80] top block unavailable for requested intra mode [h264 @ 0x56486bb46f80] error while decoding MB 45 0, bytestream 5003 [h264 @ 0x56486bdb3b00] cabac decode of qscale diff failed at 70 36 [h264 @ 0x56486bdb3b00] error while decoding MB 70 36, bytestream 7012 [h264 @ 0x56486be82980] cabac decode of qscale diff failed at 35 42 [h264 @ 0x56486", "last_exit_at": "2026-07-06T14:04:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 623.002, "last_started_at": "2026-07-06T13:54:11+09:00", "pid": 9742, "process": {"exists": false, "pid": 9742}, "restart_count": 9, "restart_count_per_hour": 6, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.741, "sample_count": 2737, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:04:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T14:04:34+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-06T14:04:34+09:00", "mjpeg": {"age_sec": 0.008, "exists": true, "mtime": "2026-07-06T14:04:34.777924+09:00", "mtime_epoch": 1783314274.7779236, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173401}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:04:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1460.254, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.737, "pushed": 3.737, "read": 3.704}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.349, "count": 112, "max_ms": 7.87, "p95_ms": 4.05}, "gstreamer_push": {"avg_ms": 1.508, "count": 112, "max_ms": 6.353, "p95_ms": 3.555}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.04, "count": 112, "max_ms": 84.877, "p95_ms": 74.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.948, "count": 112, "max_ms": 115.55, "p95_ms": 102.764}, "yolo11_640_fire_11class": {"avg_ms": 58.733, "count": 112, "max_ms": 86.379, "p95_ms": 80.43}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.698, "count": 112, "max_ms": 99.849, "p95_ms": 92.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.972, "count": 112, "max_ms": 129.32, "p95_ms": 112.851}, "read_interval": {"avg_ms": 269.758, "count": 111, "max_ms": 19317.486, "p95_ms": 130.673}, "triton_wait": {"avg_ms": 71.009, "count": 112, "max_ms": 115.564, "p95_ms": 102.78}, "visualize": {"avg_ms": 3.638, "count": 112, "max_ms": 4.571, "p95_ms": 4.245}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5538.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 10001, "restart_count": 10, "start_count": 11}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:04:34+09:00", "last_error_summary": "[h264 @ 0x56486be82980] cabac decode of qscale diff failed at 23 8 [h264 @ 0x56486be82980] error while decoding MB 23 8, bytestream 1881 [h264 @ 0x56486bb46f80] top block unavailable for requested intra mode [h264 @ 0x56486bb46f80] error while decoding MB 45 0, bytestream 5003 [h264 @ 0x56486bdb3b00] cabac decode of qscale diff failed at 70 36 [h264 @ 0x56486bdb3b00] error while decoding MB 70 36, bytestream 7012 [h264 @ 0x56486be82980] cabac decode of qscale diff failed at 35 42 [h264 @ 0x56486", "last_exit_at": "2026-07-06T14:04:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:04:37+09:00", "pid": 10001, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.699, "name": "ffmpeg", "pid": 10001, "rss_mb": 136.699, "state": "S (sleeping)", "threads": 106}, "restart_count": 10, "restart_count_per_hour": 7, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.69, "sample_count": 2692, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:04:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.725, "last_frame_pushed_at": "2026-07-06T14:04:34+09:00", "last_frame_read_age_sec": 4.814, "last_frame_read_at": "2026-07-06T14:04:34+09:00", "mjpeg": {"age_sec": 4.717, "exists": true, "mtime": "2026-07-06T14:04:34.777924+09:00", "mtime_epoch": 1783314274.7779236, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173401}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:04:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1460.254, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.067, "pushed": 2.067, "read": 2.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.452, "count": 62, "max_ms": 7.87, "p95_ms": 4.018}, "gstreamer_push": {"avg_ms": 1.541, "count": 62, "max_ms": 6.353, "p95_ms": 3.555}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.977, "count": 62, "max_ms": 84.877, "p95_ms": 76.737}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.626, "count": 62, "max_ms": 115.55, "p95_ms": 101.125}, "yolo11_640_fire_11class": {"avg_ms": 59.698, "count": 62, "max_ms": 85.473, "p95_ms": 82.409}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.763, "count": 62, "max_ms": 99.849, "p95_ms": 92.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.996, "count": 62, "max_ms": 129.32, "p95_ms": 112.847}, "read_interval": {"avg_ms": 407.603, "count": 62, "max_ms": 19317.486, "p95_ms": 131.485}, "triton_wait": {"avg_ms": 71.719, "count": 62, "max_ms": 115.564, "p95_ms": 101.137}, "visualize": {"avg_ms": 3.715, "count": 62, "max_ms": 4.571, "p95_ms": 4.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 409}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5438.977, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 15, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55826d275f00] cabac decode of qscale diff failed at 57 38 [h264 @ 0x55826d275f00] error while decoding MB 57 38, bytestream 353 [h264 @ 0x55826c9ea140] cabac decode of qscale diff failed at 5 40 [h264 @ 0x55826c9ea140] error while decoding MB 5 40, bytestream 3244 [h264 @ 0x55826c8d9200] left block unavailable for requested intra mode [h264 @ 0x55826c8d9200] error while decoding MB 0 5, bytestream 48205 [h264 @ 0x55826c901240] cabac decode of qscale diff failed at 55 41 [h264 @ 0x55826", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:16:19+09:00", "last_error_summary": "[h264 @ 0x55826d275f00] cabac decode of qscale diff failed at 57 38 [h264 @ 0x55826d275f00] error while decoding MB 57 38, bytestream 353 [h264 @ 0x55826c9ea140] cabac decode of qscale diff failed at 5 40 [h264 @ 0x55826c9ea140] error while decoding MB 5 40, bytestream 3244 [h264 @ 0x55826c8d9200] left block unavailable for requested intra mode [h264 @ 0x55826c8d9200] error while decoding MB 0 5, bytestream 48205 [h264 @ 0x55826c901240] cabac decode of qscale diff failed at 55 41 [h264 @ 0x55826", "last_exit_at": "2026-07-06T14:04:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 701.89, "last_started_at": "2026-07-06T14:04:37+09:00", "pid": 10001, "process": {"exists": false, "pid": 10001}, "restart_count": 10, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 129.043, "sample_count": 2302, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49168, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:16:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T14:16:19+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T14:16:19+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T14:16:19.329482+09:00", "mtime_epoch": 1783314979.3294823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208665}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:16:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1461.402, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.236, "pushed": 3.236, "read": 3.236}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.545, "count": 96, "max_ms": 9.734, "p95_ms": 7.006}, "gstreamer_push": {"avg_ms": 1.497, "count": 97, "max_ms": 6.743, "p95_ms": 4.068}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.537, "count": 96, "max_ms": 84.111, "p95_ms": 78.572}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.101, "count": 96, "max_ms": 123.465, "p95_ms": 103.183}, "yolo11_640_fire_11class": {"avg_ms": 61.584, "count": 96, "max_ms": 83.703, "p95_ms": 81.13}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.057, "count": 96, "max_ms": 103.909, "p95_ms": 97.669}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.809, "count": 97, "max_ms": 204.828, "p95_ms": 170.86}, "read_interval": {"avg_ms": 309.827, "count": 97, "max_ms": 18647.95, "p95_ms": 181.516}, "triton_wait": {"avg_ms": 76.399, "count": 96, "max_ms": 123.476, "p95_ms": 103.195}, "visualize": {"avg_ms": 4.418, "count": 96, "max_ms": 8.085, "p95_ms": 6.441}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 409}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5438.707, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:16:19+09:00", "last_error_summary": "[h264 @ 0x55826d275f00] cabac decode of qscale diff failed at 57 38 [h264 @ 0x55826d275f00] error while decoding MB 57 38, bytestream 353 [h264 @ 0x55826c9ea140] cabac decode of qscale diff failed at 5 40 [h264 @ 0x55826c9ea140] error while decoding MB 5 40, bytestream 3244 [h264 @ 0x55826c8d9200] left block unavailable for requested intra mode [h264 @ 0x55826c8d9200] error while decoding MB 0 5, bytestream 48205 [h264 @ 0x55826c901240] cabac decode of qscale diff failed at 55 41 [h264 @ 0x55826", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:16:19+09:00", "last_error_summary": "[h264 @ 0x55826d275f00] cabac decode of qscale diff failed at 57 38 [h264 @ 0x55826d275f00] error while decoding MB 57 38, bytestream 353 [h264 @ 0x55826c9ea140] cabac decode of qscale diff failed at 5 40 [h264 @ 0x55826c9ea140] error while decoding MB 5 40, bytestream 3244 [h264 @ 0x55826c8d9200] left block unavailable for requested intra mode [h264 @ 0x55826c8d9200] error while decoding MB 0 5, bytestream 48205 [h264 @ 0x55826c901240] cabac decode of qscale diff failed at 55 41 [h264 @ 0x55826", "last_exit_at": "2026-07-06T14:16:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 701.921, "last_started_at": "2026-07-06T14:04:37+09:00", "pid": 10001, "process": {"exists": false, "pid": 10001}, "restart_count": 10, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 129.043, "sample_count": 2302, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49169, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:16:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T14:16:19+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-06T14:16:19+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-06T14:16:19.329482+09:00", "mtime_epoch": 1783314979.3294823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208665}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:16:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1461.402, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.234, "pushed": 3.234, "read": 3.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.569, "count": 97, "max_ms": 9.734, "p95_ms": 7.006}, "gstreamer_push": {"avg_ms": 1.499, "count": 97, "max_ms": 6.743, "p95_ms": 4.068}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.514, "count": 97, "max_ms": 84.111, "p95_ms": 78.572}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.01, "count": 97, "max_ms": 123.465, "p95_ms": 103.183}, "yolo11_640_fire_11class": {"avg_ms": 61.491, "count": 97, "max_ms": 83.703, "p95_ms": 81.13}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.978, "count": 97, "max_ms": 103.909, "p95_ms": 97.669}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.187, "count": 97, "max_ms": 204.828, "p95_ms": 170.86}, "read_interval": {"avg_ms": 309.827, "count": 97, "max_ms": 18647.95, "p95_ms": 181.516}, "triton_wait": {"avg_ms": 76.305, "count": 97, "max_ms": 123.476, "p95_ms": 103.195}, "visualize": {"avg_ms": 4.428, "count": 97, "max_ms": 8.085, "p95_ms": 6.441}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 409}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5438.629, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:16:19+09:00", "last_error_summary": "[h264 @ 0x55826d275f00] cabac decode of qscale diff failed at 57 38 [h264 @ 0x55826d275f00] error while decoding MB 57 38, bytestream 353 [h264 @ 0x55826c9ea140] cabac decode of qscale diff failed at 5 40 [h264 @ 0x55826c9ea140] error while decoding MB 5 40, bytestream 3244 [h264 @ 0x55826c8d9200] left block unavailable for requested intra mode [h264 @ 0x55826c8d9200] error while decoding MB 0 5, bytestream 48205 [h264 @ 0x55826c901240] cabac decode of qscale diff failed at 55 41 [h264 @ 0x55826", "last_exit_at": "2026-07-06T14:16:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 701.932, "last_started_at": "2026-07-06T14:04:37+09:00", "pid": 10001, "process": {"exists": false, "pid": 10001}, "restart_count": 10, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 129.043, "sample_count": 2302, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49169, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:16:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-06T14:16:19+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-06T14:16:19+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-06T14:16:19.329482+09:00", "mtime_epoch": 1783314979.3294823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208665}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:16:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1461.402, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.249, "pushed": 3.249, "read": 3.215}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.569, "count": 97, "max_ms": 9.734, "p95_ms": 7.006}, "gstreamer_push": {"avg_ms": 1.499, "count": 97, "max_ms": 6.743, "p95_ms": 4.068}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.514, "count": 97, "max_ms": 84.111, "p95_ms": 78.572}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.01, "count": 97, "max_ms": 123.465, "p95_ms": 103.183}, "yolo11_640_fire_11class": {"avg_ms": 61.491, "count": 97, "max_ms": 83.703, "p95_ms": 81.13}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.978, "count": 97, "max_ms": 103.909, "p95_ms": 97.669}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.187, "count": 97, "max_ms": 204.828, "p95_ms": 170.86}, "read_interval": {"avg_ms": 311.406, "count": 96, "max_ms": 18647.95, "p95_ms": 181.516}, "triton_wait": {"avg_ms": 76.305, "count": 97, "max_ms": 123.476, "p95_ms": 103.195}, "visualize": {"avg_ms": 4.428, "count": 97, "max_ms": 8.085, "p95_ms": 6.441}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 409}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5544.266, "limit_mb": 8796093022207.996}, "detail": {"pid": 10108, "restart_count": 11, "start_count": 12}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:16:19+09:00", "last_error_summary": "[h264 @ 0x55826d275f00] cabac decode of qscale diff failed at 57 38 [h264 @ 0x55826d275f00] error while decoding MB 57 38, bytestream 353 [h264 @ 0x55826c9ea140] cabac decode of qscale diff failed at 5 40 [h264 @ 0x55826c9ea140] error while decoding MB 5 40, bytestream 3244 [h264 @ 0x55826c8d9200] left block unavailable for requested intra mode [h264 @ 0x55826c8d9200] error while decoding MB 0 5, bytestream 48205 [h264 @ 0x55826c901240] cabac decode of qscale diff failed at 55 41 [h264 @ 0x55826", "last_exit_at": "2026-07-06T14:16:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:16:21+09:00", "pid": 10108, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.594, "name": "ffmpeg", "pid": 10108, "rss_mb": 136.594, "state": "S (sleeping)", "threads": 106}, "restart_count": 11, "restart_count_per_hour": 8, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 129.097, "sample_count": 2291, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49169, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:16:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.298, "last_frame_pushed_at": "2026-07-06T14:16:19+09:00", "last_frame_read_age_sec": 4.385, "last_frame_read_at": "2026-07-06T14:16:19+09:00", "mjpeg": {"age_sec": 4.284, "exists": true, "mtime": "2026-07-06T14:16:19.421484+09:00", "mtime_epoch": 1783314979.4214845, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:16:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1461.402, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.069, "pushed": 2.069, "read": 2.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.623, "count": 62, "max_ms": 9.734, "p95_ms": 8.233}, "gstreamer_push": {"avg_ms": 1.515, "count": 62, "max_ms": 6.743, "p95_ms": 4.068}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.633, "count": 62, "max_ms": 80.587, "p95_ms": 77.718}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.594, "count": 62, "max_ms": 112.226, "p95_ms": 103.183}, "yolo11_640_fire_11class": {"avg_ms": 61.491, "count": 62, "max_ms": 82.364, "p95_ms": 77.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.262, "count": 62, "max_ms": 102.545, "p95_ms": 97.013}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.707, "count": 62, "max_ms": 204.828, "p95_ms": 170.86}, "read_interval": {"avg_ms": 421.453, "count": 61, "max_ms": 18647.95, "p95_ms": 181.516}, "triton_wait": {"avg_ms": 76.887, "count": 62, "max_ms": 112.239, "p95_ms": 103.195}, "visualize": {"avg_ms": 4.448, "count": 62, "max_ms": 8.085, "p95_ms": 6.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5396.176, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[NULL @ 0x55aef1e22200] missing picture in access unit with size 18 [h264 @ 0x55aef1feae00] no frame! [vist#0:0/h264 @ 0x55aef1e22840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:17:52+09:00", "last_error_summary": "[NULL @ 0x55aef1e22200] missing picture in access unit with size 18 [h264 @ 0x55aef1feae00] no frame! [vist#0:0/h264 @ 0x55aef1e22840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T14:00:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 1072.274, "last_started_at": "2026-07-06T14:00:00+09:00", "pid": 9849, "process": {"exists": false, "pid": 9849}, "restart_count": 14, "restart_count_per_hour": 8, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.391, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43779, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:17:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T14:17:52+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T14:17:52+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T14:17:52.883488+09:00", "mtime_epoch": 1783315072.8834882, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220292}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:17:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.836, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 287, "pushed_frames": 287, "read_frames": 287, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.588, "pushed": 9.588, "read": 9.588}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.839, "count": 287, "max_ms": 6.752, "p95_ms": 4.747}, "gstreamer_push": {"avg_ms": 1.61, "count": 287, "max_ms": 6.03, "p95_ms": 3.81}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.701, "count": 287, "max_ms": 89.626, "p95_ms": 78.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.06, "count": 287, "max_ms": 119.817, "p95_ms": 103.889}, "yolo11_640_fire_11class": {"avg_ms": 63.36, "count": 287, "max_ms": 90.1, "p95_ms": 81.253}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.25, "count": 287, "max_ms": 106.533, "p95_ms": 99.089}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.719, "count": 287, "max_ms": 136.219, "p95_ms": 117.673}, "read_interval": {"avg_ms": 104.346, "count": 287, "max_ms": 146.312, "p95_ms": 131.921}, "triton_wait": {"avg_ms": 77.503, "count": 287, "max_ms": 119.83, "p95_ms": 104.253}, "visualize": {"avg_ms": 3.652, "count": 287, "max_ms": 5.648, "p95_ms": 4.218}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5395.738, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:17:52+09:00", "last_error_summary": "[NULL @ 0x55aef1e22200] missing picture in access unit with size 18 [h264 @ 0x55aef1feae00] no frame! [vist#0:0/h264 @ 0x55aef1e22840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:17:52+09:00", "last_error_summary": "[NULL @ 0x55aef1e22200] missing picture in access unit with size 18 [h264 @ 0x55aef1feae00] no frame! [vist#0:0/h264 @ 0x55aef1e22840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T14:17:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 1072.345, "last_started_at": "2026-07-06T14:00:00+09:00", "pid": 9849, "process": {"exists": false, "pid": 9849}, "restart_count": 14, "restart_count_per_hour": 8, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.391, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43780, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:17:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T14:17:53+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T14:17:52+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-06T14:17:52.883488+09:00", "mtime_epoch": 1783315072.8834882, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220292}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:17:53+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.836, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 287, "pushed_frames": 287, "read_frames": 287, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.57, "pushed": 9.57, "read": 9.57}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.869, "count": 287, "max_ms": 12.642, "p95_ms": 5.039}, "gstreamer_push": {"avg_ms": 1.609, "count": 287, "max_ms": 6.03, "p95_ms": 3.81}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.668, "count": 287, "max_ms": 89.626, "p95_ms": 78.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.044, "count": 287, "max_ms": 119.817, "p95_ms": 103.889}, "yolo11_640_fire_11class": {"avg_ms": 63.343, "count": 287, "max_ms": 90.1, "p95_ms": 81.253}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.244, "count": 287, "max_ms": 106.533, "p95_ms": 99.089}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.885, "count": 287, "max_ms": 136.219, "p95_ms": 118.857}, "read_interval": {"avg_ms": 104.346, "count": 287, "max_ms": 146.312, "p95_ms": 131.921}, "triton_wait": {"avg_ms": 77.487, "count": 287, "max_ms": 119.83, "p95_ms": 104.253}, "visualize": {"avg_ms": 3.673, "count": 287, "max_ms": 9.493, "p95_ms": 4.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5395.73, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:17:52+09:00", "last_error_summary": "[NULL @ 0x55aef1e22200] missing picture in access unit with size 18 [h264 @ 0x55aef1feae00] no frame! [vist#0:0/h264 @ 0x55aef1e22840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T14:17:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 1072.352, "last_started_at": "2026-07-06T14:00:00+09:00", "pid": 9849, "process": {"exists": false, "pid": 9849}, "restart_count": 14, "restart_count_per_hour": 8, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.391, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43780, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:17:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T14:17:53+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-06T14:17:52+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-06T14:17:52.883488+09:00", "mtime_epoch": 1783315072.8834882, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220292}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:17:53+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.836, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 287, "pushed_frames": 287, "read_frames": 287, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.568, "pushed": 9.568, "read": 9.568}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.869, "count": 287, "max_ms": 12.642, "p95_ms": 5.039}, "gstreamer_push": {"avg_ms": 1.609, "count": 287, "max_ms": 6.03, "p95_ms": 3.81}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.668, "count": 287, "max_ms": 89.626, "p95_ms": 78.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.044, "count": 287, "max_ms": 119.817, "p95_ms": 103.889}, "yolo11_640_fire_11class": {"avg_ms": 63.343, "count": 287, "max_ms": 90.1, "p95_ms": 81.253}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.244, "count": 287, "max_ms": 106.533, "p95_ms": 99.089}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.885, "count": 287, "max_ms": 136.219, "p95_ms": 118.857}, "read_interval": {"avg_ms": 104.346, "count": 287, "max_ms": 146.312, "p95_ms": 131.921}, "triton_wait": {"avg_ms": 77.487, "count": 287, "max_ms": 119.83, "p95_ms": 104.253}, "visualize": {"avg_ms": 3.673, "count": 287, "max_ms": 9.493, "p95_ms": 4.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5511.164, "limit_mb": 8796093022207.996}, "detail": {"pid": 10215, "restart_count": 15, "start_count": 16}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:17:52+09:00", "last_error_summary": "[NULL @ 0x55aef1e22200] missing picture in access unit with size 18 [h264 @ 0x55aef1feae00] no frame! [vist#0:0/h264 @ 0x55aef1e22840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T14:17:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:17:55+09:00", "pid": 10215, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.84, "name": "ffmpeg", "pid": 10215, "rss_mb": 146.84, "state": "R (running)", "threads": 49}, "restart_count": 15, "restart_count_per_hour": 9, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.933, "sample_count": 2464, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43780, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:17:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.742, "last_frame_pushed_at": "2026-07-06T14:17:53+09:00", "last_frame_read_age_sec": 4.869, "last_frame_read_at": "2026-07-06T14:17:52+09:00", "mjpeg": {"age_sec": 4.723, "exists": true, "mtime": "2026-07-06T14:17:53.023491+09:00", "mtime_epoch": 1783315073.0234911, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228571}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:17:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.836, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 239, "pushed_frames": 239, "read_frames": 238, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.98, "pushed": 7.98, "read": 7.946}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.905, "count": 239, "max_ms": 12.642, "p95_ms": 5.405}, "gstreamer_push": {"avg_ms": 1.555, "count": 239, "max_ms": 6.03, "p95_ms": 3.787}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.895, "count": 239, "max_ms": 89.626, "p95_ms": 79.352}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.331, "count": 239, "max_ms": 119.817, "p95_ms": 104.246}, "yolo11_640_fire_11class": {"avg_ms": 64.344, "count": 239, "max_ms": 90.1, "p95_ms": 82.87}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.118, "count": 239, "max_ms": 106.533, "p95_ms": 99.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.323, "count": 239, "max_ms": 136.219, "p95_ms": 119.687}, "read_interval": {"avg_ms": 105.818, "count": 238, "max_ms": 146.312, "p95_ms": 133.194}, "triton_wait": {"avg_ms": 78.671, "count": 239, "max_ms": 119.83, "p95_ms": 104.766}, "visualize": {"avg_ms": 3.706, "count": 239, "max_ms": 9.493, "p95_ms": 4.359}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 416}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5445.27, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 18, bytestream 14411 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 24, bytestream 5806 [h264 @ 0x5635a7d1c900] cabac decode of qscale diff failed at 5 44 [h264 @ 0x5635a7d1c900] error while decoding MB 5 44, bytestream 6591 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:18:38+09:00", "last_error_summary": "[h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 18, bytestream 14411 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 24, bytestream 5806 [h264 @ 0x5635a7d1c900] cabac decode of qscale diff failed at 5 44 [h264 @ 0x5635a7d1c900] error while decoding MB 5 44, bytestream 6591 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra", "last_exit_at": "2026-07-06T14:16:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.048, "last_started_at": "2026-07-06T14:16:21+09:00", "pid": 10108, "process": {"exists": false, "pid": 10108}, "restart_count": 11, "restart_count_per_hour": 8, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.954, "sample_count": 2184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50037, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:18:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T14:18:38+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T14:18:38+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T14:18:38.708468+09:00", "mtime_epoch": 1783315118.7084677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210136}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:18:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1463.664, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 103, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.439, "pushed": 3.439, "read": 3.439}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.523, "count": 103, "max_ms": 5.569, "p95_ms": 4.735}, "gstreamer_push": {"avg_ms": 1.807, "count": 103, "max_ms": 6.492, "p95_ms": 5.838}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.155, "count": 103, "max_ms": 90.812, "p95_ms": 84.849}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.999, "count": 103, "max_ms": 112.76, "p95_ms": 107.681}, "yolo11_640_fire_11class": {"avg_ms": 67.735, "count": 103, "max_ms": 96.547, "p95_ms": 88.797}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.498, "count": 103, "max_ms": 105.684, "p95_ms": 98.845}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.811, "count": 103, "max_ms": 124.914, "p95_ms": 119.868}, "read_interval": {"avg_ms": 291.463, "count": 103, "max_ms": 19268.303, "p95_ms": 133.574}, "triton_wait": {"avg_ms": 80.277, "count": 103, "max_ms": 112.772, "p95_ms": 107.694}, "visualize": {"avg_ms": 3.76, "count": 103, "max_ms": 6.269, "p95_ms": 4.404}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 416}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5444.848, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:18:38+09:00", "last_error_summary": "[h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 18, bytestream 14411 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 24, bytestream 5806 [h264 @ 0x5635a7d1c900] cabac decode of qscale diff failed at 5 44 [h264 @ 0x5635a7d1c900] error while decoding MB 5 44, bytestream 6591 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:18:38+09:00", "last_error_summary": "[h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 18, bytestream 14411 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 24, bytestream 5806 [h264 @ 0x5635a7d1c900] cabac decode of qscale diff failed at 5 44 [h264 @ 0x5635a7d1c900] error while decoding MB 5 44, bytestream 6591 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra", "last_exit_at": "2026-07-06T14:18:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.079, "last_started_at": "2026-07-06T14:16:21+09:00", "pid": 10108, "process": {"exists": false, "pid": 10108}, "restart_count": 11, "restart_count_per_hour": 8, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.954, "sample_count": 2185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50038, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:18:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T14:18:38+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-06T14:18:38+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-06T14:18:38.708468+09:00", "mtime_epoch": 1783315118.7084677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210136}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:18:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1463.664, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.468, "pushed": 3.468, "read": 3.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.52, "count": 104, "max_ms": 5.569, "p95_ms": 4.735}, "gstreamer_push": {"avg_ms": 1.803, "count": 104, "max_ms": 6.492, "p95_ms": 5.838}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.034, "count": 104, "max_ms": 90.812, "p95_ms": 84.849}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.845, "count": 104, "max_ms": 112.76, "p95_ms": 107.681}, "yolo11_640_fire_11class": {"avg_ms": 67.623, "count": 104, "max_ms": 96.547, "p95_ms": 88.797}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.393, "count": 104, "max_ms": 105.684, "p95_ms": 98.845}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.647, "count": 104, "max_ms": 124.914, "p95_ms": 119.868}, "read_interval": {"avg_ms": 291.463, "count": 103, "max_ms": 19268.303, "p95_ms": 133.574}, "triton_wait": {"avg_ms": 80.121, "count": 104, "max_ms": 112.772, "p95_ms": 107.694}, "visualize": {"avg_ms": 3.758, "count": 104, "max_ms": 6.269, "p95_ms": 4.404}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 416}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5444.961, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:18:38+09:00", "last_error_summary": "[h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 18, bytestream 14411 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 24, bytestream 5806 [h264 @ 0x5635a7d1c900] cabac decode of qscale diff failed at 5 44 [h264 @ 0x5635a7d1c900] error while decoding MB 5 44, bytestream 6591 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra", "last_exit_at": "2026-07-06T14:18:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.087, "last_started_at": "2026-07-06T14:16:21+09:00", "pid": 10108, "process": {"exists": false, "pid": 10108}, "restart_count": 11, "restart_count_per_hour": 8, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.954, "sample_count": 2185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50038, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:18:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T14:18:38+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-06T14:18:38+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-06T14:18:38.708468+09:00", "mtime_epoch": 1783315118.7084677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210136}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:18:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1463.664, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.468, "pushed": 3.468, "read": 3.434}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.52, "count": 104, "max_ms": 5.569, "p95_ms": 4.735}, "gstreamer_push": {"avg_ms": 1.803, "count": 104, "max_ms": 6.492, "p95_ms": 5.838}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.034, "count": 104, "max_ms": 90.812, "p95_ms": 84.849}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.845, "count": 104, "max_ms": 112.76, "p95_ms": 107.681}, "yolo11_640_fire_11class": {"avg_ms": 67.623, "count": 104, "max_ms": 96.547, "p95_ms": 88.797}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.393, "count": 104, "max_ms": 105.684, "p95_ms": 98.845}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.647, "count": 104, "max_ms": 124.914, "p95_ms": 119.868}, "read_interval": {"avg_ms": 291.463, "count": 103, "max_ms": 19268.303, "p95_ms": 133.574}, "triton_wait": {"avg_ms": 80.121, "count": 104, "max_ms": 112.772, "p95_ms": 107.694}, "visualize": {"avg_ms": 3.758, "count": 104, "max_ms": 6.269, "p95_ms": 4.404}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 416}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5544.113, "limit_mb": 8796093022207.996}, "detail": {"pid": 10367, "restart_count": 12, "start_count": 13}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:18:38+09:00", "last_error_summary": "[h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 18, bytestream 14411 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5635a7d30ac0] error while decoding MB 0 24, bytestream 5806 [h264 @ 0x5635a7d1c900] cabac decode of qscale diff failed at 5 44 [h264 @ 0x5635a7d1c900] error while decoding MB 5 44, bytestream 6591 [h264 @ 0x5635a7d30ac0] left block unavailable for requested intra", "last_exit_at": "2026-07-06T14:18:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:18:41+09:00", "pid": 10367, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.102, "name": "ffmpeg", "pid": 10367, "rss_mb": 139.102, "state": "S (sleeping)", "threads": 106}, "restart_count": 12, "restart_count_per_hour": 9, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.01, "sample_count": 2146, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50038, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:18:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.386, "last_frame_pushed_at": "2026-07-06T14:18:38+09:00", "last_frame_read_age_sec": 4.464, "last_frame_read_at": "2026-07-06T14:18:38+09:00", "mjpeg": {"age_sec": 4.367, "exists": true, "mtime": "2026-07-06T14:18:38.796470+09:00", "mtime_epoch": 1783315118.7964697, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209988}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:18:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1463.664, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.006, "pushed": 2.006, "read": 1.972}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.462, "count": 60, "max_ms": 5.569, "p95_ms": 4.021}, "gstreamer_push": {"avg_ms": 1.826, "count": 60, "max_ms": 6.492, "p95_ms": 4.03}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.531, "count": 60, "max_ms": 90.812, "p95_ms": 85.106}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.966, "count": 60, "max_ms": 112.76, "p95_ms": 107.681}, "yolo11_640_fire_11class": {"avg_ms": 69.0, "count": 60, "max_ms": 89.82, "p95_ms": 86.153}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.843, "count": 60, "max_ms": 104.6, "p95_ms": 97.233}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.574, "count": 60, "max_ms": 124.914, "p95_ms": 119.868}, "read_interval": {"avg_ms": 433.105, "count": 59, "max_ms": 19268.303, "p95_ms": 135.526}, "triton_wait": {"avg_ms": 83.09, "count": 60, "max_ms": 112.772, "p95_ms": 107.694}, "visualize": {"avg_ms": 3.731, "count": 60, "max_ms": 6.269, "p95_ms": 3.953}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5383.023, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5648cdebb000] error while decoding MB 6 40, bytestream -5 [in#0/rtsp @ 0x5648cde73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cde73e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:21:40+09:00", "last_error_summary": "[h264 @ 0x5648cdebb000] error while decoding MB 6 40, bytestream -5 [in#0/rtsp @ 0x5648cde73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cde73e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:17:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.778, "last_started_at": "2026-07-06T14:17:55+09:00", "pid": 10215, "process": {"exists": false, "pid": 10215}, "restart_count": 15, "restart_count_per_hour": 7, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.719, "sample_count": 2375, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45442, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:21:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T14:21:40+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T14:21:40+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-06T14:21:40.472339+09:00", "mtime_epoch": 1783315300.4723387, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:21:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1326.77, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 57, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.903, "pushed": 1.903, "read": 1.903}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.245, "count": 57, "max_ms": 7.866, "p95_ms": 6.068}, "gstreamer_push": {"avg_ms": 1.374, "count": 57, "max_ms": 2.76, "p95_ms": 2.544}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.692, "count": 57, "max_ms": 79.363, "p95_ms": 76.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.434, "count": 57, "max_ms": 108.112, "p95_ms": 105.495}, "yolo11_640_fire_11class": {"avg_ms": 59.137, "count": 57, "max_ms": 84.4, "p95_ms": 82.222}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.167, "count": 57, "max_ms": 101.627, "p95_ms": 98.626}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.933, "count": 57, "max_ms": 163.172, "p95_ms": 159.94}, "read_interval": {"avg_ms": 526.168, "count": 57, "max_ms": 13286.638, "p95_ms": 172.522}, "triton_wait": {"avg_ms": 69.979, "count": 57, "max_ms": 108.125, "p95_ms": 105.508}, "visualize": {"avg_ms": 4.64, "count": 57, "max_ms": 7.783, "p95_ms": 5.321}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5382.562, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:21:40+09:00", "last_error_summary": "[h264 @ 0x5648cdebb000] error while decoding MB 6 40, bytestream -5 [in#0/rtsp @ 0x5648cde73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cde73e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:21:40+09:00", "last_error_summary": "[h264 @ 0x5648cdebb000] error while decoding MB 6 40, bytestream -5 [in#0/rtsp @ 0x5648cde73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cde73e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:21:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.883, "last_started_at": "2026-07-06T14:17:55+09:00", "pid": 10215, "process": {"exists": false, "pid": 10215}, "restart_count": 15, "restart_count_per_hour": 7, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.749, "sample_count": 2375, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:21:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T14:21:40+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-06T14:21:40+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-06T14:21:40.472339+09:00", "mtime_epoch": 1783315300.4723387, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:21:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1326.77, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.908, "pushed": 1.908, "read": 1.874}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.241, "count": 57, "max_ms": 7.866, "p95_ms": 6.068}, "gstreamer_push": {"avg_ms": 1.37, "count": 57, "max_ms": 2.76, "p95_ms": 2.544}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.168, "count": 57, "max_ms": 79.363, "p95_ms": 76.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.438, "count": 57, "max_ms": 112.359, "p95_ms": 105.979}, "yolo11_640_fire_11class": {"avg_ms": 59.789, "count": 57, "max_ms": 85.906, "p95_ms": 84.061}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.715, "count": 57, "max_ms": 101.627, "p95_ms": 98.626}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.917, "count": 57, "max_ms": 163.172, "p95_ms": 160.217}, "read_interval": {"avg_ms": 533.455, "count": 56, "max_ms": 13286.638, "p95_ms": 172.522}, "triton_wait": {"avg_ms": 70.984, "count": 57, "max_ms": 112.37, "p95_ms": 105.991}, "visualize": {"avg_ms": 4.65, "count": 57, "max_ms": 7.783, "p95_ms": 5.321}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5382.555, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:21:40+09:00", "last_error_summary": "[h264 @ 0x5648cdebb000] error while decoding MB 6 40, bytestream -5 [in#0/rtsp @ 0x5648cde73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cde73e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:21:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.888, "last_started_at": "2026-07-06T14:17:55+09:00", "pid": 10215, "process": {"exists": false, "pid": 10215}, "restart_count": 15, "restart_count_per_hour": 7, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.749, "sample_count": 2375, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:21:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T14:21:40+09:00", "last_frame_read_age_sec": 0.172, "last_frame_read_at": "2026-07-06T14:21:40+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-06T14:21:40.472339+09:00", "mtime_epoch": 1783315300.4723387, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:21:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1326.77, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.907, "pushed": 1.907, "read": 1.874}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.241, "count": 57, "max_ms": 7.866, "p95_ms": 6.068}, "gstreamer_push": {"avg_ms": 1.37, "count": 57, "max_ms": 2.76, "p95_ms": 2.544}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.168, "count": 57, "max_ms": 79.363, "p95_ms": 76.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.438, "count": 57, "max_ms": 112.359, "p95_ms": 105.979}, "yolo11_640_fire_11class": {"avg_ms": 59.789, "count": 57, "max_ms": 85.906, "p95_ms": 84.061}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.715, "count": 57, "max_ms": 101.627, "p95_ms": 98.626}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.917, "count": 57, "max_ms": 163.172, "p95_ms": 160.217}, "read_interval": {"avg_ms": 533.455, "count": 56, "max_ms": 13286.638, "p95_ms": 172.522}, "triton_wait": {"avg_ms": 70.984, "count": 57, "max_ms": 112.37, "p95_ms": 105.991}, "visualize": {"avg_ms": 4.65, "count": 57, "max_ms": 7.783, "p95_ms": 5.321}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5550.57, "limit_mb": 8796093022207.996}, "detail": {"pid": 10474, "restart_count": 16, "start_count": 17}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:21:40+09:00", "last_error_summary": "[h264 @ 0x5648cdebb000] error while decoding MB 6 40, bytestream -5 [in#0/rtsp @ 0x5648cde73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cde73e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:21:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:21:43+09:00", "pid": 10474, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.938, "name": "ffmpeg", "pid": 10474, "rss_mb": 194.938, "state": "S (sleeping)", "threads": 106}, "restart_count": 16, "restart_count_per_hour": 8, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.719, "sample_count": 2336, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:21:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.575, "last_frame_pushed_at": "2026-07-06T14:21:40+09:00", "last_frame_read_age_sec": 4.738, "last_frame_read_at": "2026-07-06T14:21:40+09:00", "mjpeg": {"age_sec": 4.559, "exists": true, "mtime": "2026-07-06T14:21:40.640342+09:00", "mtime_epoch": 1783315300.6403422, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229075}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:21:45+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1326.77, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 17, "pushed_frames": 17, "read_frames": 16, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.568, "pushed": 0.568, "read": 0.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.524, "count": 17, "max_ms": 6.232, "p95_ms": 6.232}, "gstreamer_push": {"avg_ms": 1.414, "count": 17, "max_ms": 2.76, "p95_ms": 2.76}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.06, "count": 17, "max_ms": 76.435, "p95_ms": 76.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.631, "count": 17, "max_ms": 112.359, "p95_ms": 112.359}, "yolo11_640_fire_11class": {"avg_ms": 62.199, "count": 17, "max_ms": 85.906, "p95_ms": 85.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.516, "count": 17, "max_ms": 101.627, "p95_ms": 101.627}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.208, "count": 17, "max_ms": 163.172, "p95_ms": 163.172}, "read_interval": {"avg_ms": 1579.407, "count": 16, "max_ms": 13286.638, "p95_ms": 13286.638}, "triton_wait": {"avg_ms": 75.094, "count": 17, "max_ms": 112.37, "p95_ms": 112.37}, "visualize": {"avg_ms": 4.849, "count": 17, "max_ms": 5.253, "p95_ms": 5.253}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 382}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5386.773, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x5633b628e200] missing picture in access unit with size 18 [h264 @ 0x5633b6dfc580] no frame! [in#0/rtsp @ 0x5633b6289e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5633b6289e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5633b628e840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:23:11+09:00", "last_error_summary": "[NULL @ 0x5633b628e200] missing picture in access unit with size 18 [h264 @ 0x5633b6dfc580] no frame! [in#0/rtsp @ 0x5633b6289e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5633b6289e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5633b628e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T14:21:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 87.824, "last_started_at": "2026-07-06T14:21:43+09:00", "pid": 10474, "process": {"exists": false, "pid": 10474}, "restart_count": 16, "restart_count_per_hour": 7, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.545, "sample_count": 1951, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45843, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:23:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-06T14:23:10+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-06T14:23:10+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-06T14:23:10.958259+09:00", "mtime_epoch": 1783315390.9582586, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229714}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:23:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1320.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.437, "pushed": 0.437, "read": 0.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.684, "count": 12, "max_ms": 6.684, "p95_ms": 6.684}, "gstreamer_push": {"avg_ms": 1.649, "count": 12, "max_ms": 5.121, "p95_ms": 5.121}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.918, "count": 12, "max_ms": 65.702, "p95_ms": 65.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.671, "count": 12, "max_ms": 95.603, "p95_ms": 95.603}, "yolo11_640_fire_11class": {"avg_ms": 57.76, "count": 12, "max_ms": 67.333, "p95_ms": 67.333}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.095, "count": 12, "max_ms": 84.906, "p95_ms": 84.906}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.058, "count": 12, "max_ms": 129.335, "p95_ms": 129.335}, "read_interval": {"avg_ms": 2339.26, "count": 13, "max_ms": 26264.529, "p95_ms": 26264.529}, "triton_wait": {"avg_ms": 71.927, "count": 12, "max_ms": 95.615, "p95_ms": 95.615}, "visualize": {"avg_ms": 3.945, "count": 12, "max_ms": 5.187, "p95_ms": 5.187}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.474, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 382}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5386.598, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:23:11+09:00", "last_error_summary": "[NULL @ 0x5633b628e200] missing picture in access unit with size 18 [h264 @ 0x5633b6dfc580] no frame! [in#0/rtsp @ 0x5633b6289e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5633b6289e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5633b628e840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:23:11+09:00", "last_error_summary": "[NULL @ 0x5633b628e200] missing picture in access unit with size 18 [h264 @ 0x5633b6dfc580] no frame! [in#0/rtsp @ 0x5633b6289e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5633b6289e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5633b628e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T14:23:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 87.854, "last_started_at": "2026-07-06T14:21:43+09:00", "pid": 10474, "process": {"exists": false, "pid": 10474}, "restart_count": 16, "restart_count_per_hour": 7, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.545, "sample_count": 1951, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45844, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:23:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T14:23:11+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-06T14:23:10+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-06T14:23:10.958259+09:00", "mtime_epoch": 1783315390.9582586, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229714}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:23:11+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1320.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.473, "pushed": 0.473, "read": 0.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.012, "count": 13, "max_ms": 8.947, "p95_ms": 8.947}, "gstreamer_push": {"avg_ms": 1.614, "count": 13, "max_ms": 5.121, "p95_ms": 5.121}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.804, "count": 13, "max_ms": 65.702, "p95_ms": 65.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.67, "count": 13, "max_ms": 95.603, "p95_ms": 95.603}, "yolo11_640_fire_11class": {"avg_ms": 58.738, "count": 13, "max_ms": 70.482, "p95_ms": 70.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.587, "count": 13, "max_ms": 84.906, "p95_ms": 84.906}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.71, "count": 13, "max_ms": 129.335, "p95_ms": 129.335}, "read_interval": {"avg_ms": 2339.26, "count": 13, "max_ms": 26264.529, "p95_ms": 26264.529}, "triton_wait": {"avg_ms": 71.907, "count": 13, "max_ms": 95.615, "p95_ms": 95.615}, "visualize": {"avg_ms": 3.928, "count": 13, "max_ms": 5.187, "p95_ms": 5.187}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.503, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 382}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5386.582, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:23:11+09:00", "last_error_summary": "[NULL @ 0x5633b628e200] missing picture in access unit with size 18 [h264 @ 0x5633b6dfc580] no frame! [in#0/rtsp @ 0x5633b6289e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5633b6289e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5633b628e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T14:23:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 87.861, "last_started_at": "2026-07-06T14:21:43+09:00", "pid": 10474, "process": {"exists": false, "pid": 10474}, "restart_count": 16, "restart_count_per_hour": 7, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.545, "sample_count": 1951, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45844, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:23:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T14:23:11+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-06T14:23:10+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T14:23:10.958259+09:00", "mtime_epoch": 1783315390.9582586, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229714}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:23:11+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1320.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.473, "pushed": 0.473, "read": 0.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.012, "count": 13, "max_ms": 8.947, "p95_ms": 8.947}, "gstreamer_push": {"avg_ms": 1.614, "count": 13, "max_ms": 5.121, "p95_ms": 5.121}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.804, "count": 13, "max_ms": 65.702, "p95_ms": 65.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.67, "count": 13, "max_ms": 95.603, "p95_ms": 95.603}, "yolo11_640_fire_11class": {"avg_ms": 58.738, "count": 13, "max_ms": 70.482, "p95_ms": 70.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.587, "count": 13, "max_ms": 84.906, "p95_ms": 84.906}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.71, "count": 13, "max_ms": 129.335, "p95_ms": 129.335}, "read_interval": {"avg_ms": 2339.26, "count": 13, "max_ms": 26264.529, "p95_ms": 26264.529}, "triton_wait": {"avg_ms": 71.907, "count": 13, "max_ms": 95.615, "p95_ms": 95.615}, "visualize": {"avg_ms": 3.928, "count": 13, "max_ms": 5.187, "p95_ms": 5.187}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.51, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 382}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5408.496, "limit_mb": 8796093022207.996}, "detail": {"pid": 10626, "restart_count": 17, "start_count": 18}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:23:11+09:00", "last_error_summary": "[NULL @ 0x5633b628e200] missing picture in access unit with size 18 [h264 @ 0x5633b6dfc580] no frame! [in#0/rtsp @ 0x5633b6289e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5633b6289e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5633b628e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T14:23:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:23:13+09:00", "pid": 10626, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.168, "name": "ffmpeg", "pid": 10626, "rss_mb": 58.168, "state": "S (sleeping)", "threads": 1}, "restart_count": 17, "restart_count_per_hour": 8, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.599, "sample_count": 1914, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45844, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:23:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.314, "last_frame_pushed_at": "2026-07-06T14:23:11+09:00", "last_frame_read_age_sec": 4.406, "last_frame_read_at": "2026-07-06T14:23:10+09:00", "mjpeg": {"age_sec": 4.294, "exists": true, "mtime": "2026-07-06T14:23:11.066261+09:00", "mtime_epoch": 1783315391.0662608, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 225656}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:23:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1320.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.163, "pushed": 2.163, "read": 2.163}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.872, "count": 12, "max_ms": 8.947, "p95_ms": 8.947}, "gstreamer_push": {"avg_ms": 1.662, "count": 12, "max_ms": 5.121, "p95_ms": 5.121}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.412, "count": 12, "max_ms": 65.702, "p95_ms": 65.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.648, "count": 12, "max_ms": 95.603, "p95_ms": 95.603}, "yolo11_640_fire_11class": {"avg_ms": 58.874, "count": 12, "max_ms": 70.482, "p95_ms": 70.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.823, "count": 12, "max_ms": 84.906, "p95_ms": 84.906}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.003, "count": 12, "max_ms": 129.335, "p95_ms": 129.335}, "read_interval": {"avg_ms": 2283.81, "count": 12, "max_ms": 26264.529, "p95_ms": 26264.529}, "triton_wait": {"avg_ms": 72.751, "count": 12, "max_ms": 95.615, "p95_ms": 95.615}, "visualize": {"avg_ms": 3.823, "count": 12, "max_ms": 5.176, "p95_ms": 5.176}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.547, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5408.184, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55688413b700] error while decoding MB 64 34, bytestream -15 [in#0/rtsp @ 0x55688342be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55688342be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:25:06+09:00", "last_error_summary": "[h264 @ 0x55688413b700] error while decoding MB 64 34, bytestream -15 [in#0/rtsp @ 0x55688342be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55688342be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:23:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.713, "last_started_at": "2026-07-06T14:23:13+09:00", "pid": 10626, "process": {"exists": false, "pid": 10626}, "restart_count": 17, "restart_count_per_hour": 8, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.966, "sample_count": 1629, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46503, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:25:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T14:25:05+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-06T14:25:06+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T14:25:06.016187+09:00", "mtime_epoch": 1783315506.016187, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224533}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:25:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1325.23, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.727, "pushed": 0.727, "read": 0.783}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.243, "count": 13, "max_ms": 9.672, "p95_ms": 9.672}, "gstreamer_push": {"avg_ms": 1.378, "count": 13, "max_ms": 2.491, "p95_ms": 2.491}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.156, "count": 13, "max_ms": 59.194, "p95_ms": 59.194}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.897, "count": 13, "max_ms": 100.125, "p95_ms": 100.125}, "yolo11_640_fire_11class": {"avg_ms": 58.538, "count": 13, "max_ms": 74.433, "p95_ms": 74.433}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.451, "count": 13, "max_ms": 89.896, "p95_ms": 89.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.457, "count": 13, "max_ms": 156.452, "p95_ms": 156.452}, "read_interval": {"avg_ms": 2367.899, "count": 14, "max_ms": 16322.191, "p95_ms": 16322.191}, "triton_wait": {"avg_ms": 69.496, "count": 13, "max_ms": 100.137, "p95_ms": 100.137}, "visualize": {"avg_ms": 5.348, "count": 13, "max_ms": 7.884, "p95_ms": 7.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.888, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5408.227, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:25:06+09:00", "last_error_summary": "[h264 @ 0x55688413b700] error while decoding MB 64 34, bytestream -15 [in#0/rtsp @ 0x55688342be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55688342be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:25:06+09:00", "last_error_summary": "[h264 @ 0x55688413b700] error while decoding MB 64 34, bytestream -15 [in#0/rtsp @ 0x55688342be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55688342be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:25:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.731, "last_started_at": "2026-07-06T14:23:13+09:00", "pid": 10626, "process": {"exists": false, "pid": 10626}, "restart_count": 17, "restart_count_per_hour": 8, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.966, "sample_count": 1630, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:25:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T14:25:06+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-06T14:25:06+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-06T14:25:06.016187+09:00", "mtime_epoch": 1783315506.016187, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224533}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:25:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1325.23, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.066, "count": 14, "max_ms": 9.672, "p95_ms": 9.672}, "gstreamer_push": {"avg_ms": 1.385, "count": 14, "max_ms": 2.491, "p95_ms": 2.491}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.462, "count": 14, "max_ms": 59.194, "p95_ms": 59.194}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.267, "count": 14, "max_ms": 100.125, "p95_ms": 100.125}, "yolo11_640_fire_11class": {"avg_ms": 58.412, "count": 14, "max_ms": 74.433, "p95_ms": 74.433}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.811, "count": 14, "max_ms": 89.896, "p95_ms": 89.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.716, "count": 14, "max_ms": 156.452, "p95_ms": 156.452}, "read_interval": {"avg_ms": 2367.899, "count": 14, "max_ms": 16322.191, "p95_ms": 16322.191}, "triton_wait": {"avg_ms": 68.824, "count": 14, "max_ms": 100.137, "p95_ms": 100.137}, "visualize": {"avg_ms": 5.339, "count": 14, "max_ms": 7.884, "p95_ms": 7.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.905, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5407.605, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:25:06+09:00", "last_error_summary": "[h264 @ 0x55688413b700] error while decoding MB 64 34, bytestream -15 [in#0/rtsp @ 0x55688342be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55688342be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:25:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.739, "last_started_at": "2026-07-06T14:23:13+09:00", "pid": 10626, "process": {"exists": false, "pid": 10626}, "restart_count": 17, "restart_count_per_hour": 8, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.966, "sample_count": 1630, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:25:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T14:25:06+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-06T14:25:06+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-06T14:25:06.016187+09:00", "mtime_epoch": 1783315506.016187, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224533}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:25:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1325.23, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.066, "count": 14, "max_ms": 9.672, "p95_ms": 9.672}, "gstreamer_push": {"avg_ms": 1.385, "count": 14, "max_ms": 2.491, "p95_ms": 2.491}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.462, "count": 14, "max_ms": 59.194, "p95_ms": 59.194}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.267, "count": 14, "max_ms": 100.125, "p95_ms": 100.125}, "yolo11_640_fire_11class": {"avg_ms": 58.412, "count": 14, "max_ms": 74.433, "p95_ms": 74.433}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.811, "count": 14, "max_ms": 89.896, "p95_ms": 89.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.716, "count": 14, "max_ms": 156.452, "p95_ms": 156.452}, "read_interval": {"avg_ms": 2367.899, "count": 14, "max_ms": 16322.191, "p95_ms": 16322.191}, "triton_wait": {"avg_ms": 68.824, "count": 14, "max_ms": 100.137, "p95_ms": 100.137}, "visualize": {"avg_ms": 5.339, "count": 14, "max_ms": 7.884, "p95_ms": 7.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.913, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5431.973, "limit_mb": 8796093022207.996}, "detail": {"pid": 10778, "restart_count": 18, "start_count": 19}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:25:06+09:00", "last_error_summary": "[h264 @ 0x55688413b700] error while decoding MB 64 34, bytestream -15 [in#0/rtsp @ 0x55688342be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55688342be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:25:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:25:08+09:00", "pid": 10778, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.242, "name": "ffmpeg", "pid": 10778, "rss_mb": 58.242, "state": "S (sleeping)", "threads": 1}, "restart_count": 18, "restart_count_per_hour": 9, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.573, "sample_count": 1586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:25:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.294, "last_frame_pushed_at": "2026-07-06T14:25:06+09:00", "last_frame_read_age_sec": 4.373, "last_frame_read_at": "2026-07-06T14:25:06+09:00", "mjpeg": {"age_sec": 4.278, "exists": true, "mtime": "2026-07-06T14:25:06.100187+09:00", "mtime_epoch": 1783315506.1001866, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228058}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:25:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1325.23, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.631, "pushed": 0.631, "read": 0.631}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.066, "count": 14, "max_ms": 9.672, "p95_ms": 9.672}, "gstreamer_push": {"avg_ms": 1.385, "count": 14, "max_ms": 2.491, "p95_ms": 2.491}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.462, "count": 14, "max_ms": 59.194, "p95_ms": 59.194}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.267, "count": 14, "max_ms": 100.125, "p95_ms": 100.125}, "yolo11_640_fire_11class": {"avg_ms": 58.412, "count": 14, "max_ms": 74.433, "p95_ms": 74.433}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.811, "count": 14, "max_ms": 89.896, "p95_ms": 89.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.716, "count": 14, "max_ms": 156.452, "p95_ms": 156.452}, "read_interval": {"avg_ms": 2367.899, "count": 14, "max_ms": 16322.191, "p95_ms": 16322.191}, "triton_wait": {"avg_ms": 68.824, "count": 14, "max_ms": 100.137, "p95_ms": 100.137}, "visualize": {"avg_ms": 5.339, "count": 14, "max_ms": 7.884, "p95_ms": 7.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.193, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 448}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5447.746, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5584f31b37c0] left block unavailable for requested intra mode [h264 @ 0x5584f31b37c0] error while decoding MB 0 27, bytestream 48619 [h264 @ 0x5584f27b4000] reference picture missing during reorder [h264 @ 0x5584f27b4000] Missing reference picture, default is 65584 [h264 @ 0x5584f27b69c0] mmco: unref short failure [h264 @ 0x5584f27b69c0] illegal short term buffer state detected [h264 @ 0x5584f34ef1c0] top block unavailable for requested intra mode -1 [h264 @ 0x5584f34ef1c0] error while", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:25:47+09:00", "last_error_summary": "[h264 @ 0x5584f31b37c0] left block unavailable for requested intra mode [h264 @ 0x5584f31b37c0] error while decoding MB 0 27, bytestream 48619 [h264 @ 0x5584f27b4000] reference picture missing during reorder [h264 @ 0x5584f27b4000] Missing reference picture, default is 65584 [h264 @ 0x5584f27b69c0] mmco: unref short failure [h264 @ 0x5584f27b69c0] illegal short term buffer state detected [h264 @ 0x5584f34ef1c0] top block unavailable for requested intra mode -1 [h264 @ 0x5584f34ef1c0] error while", "last_exit_at": "2026-07-06T14:18:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 426.672, "last_started_at": "2026-07-06T14:18:41+09:00", "pid": 10367, "process": {"exists": false, "pid": 10367}, "restart_count": 12, "restart_count_per_hour": 8, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.514, "sample_count": 2670, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53791, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:25:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T14:25:47+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-06T14:25:47+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T14:25:47.788013+09:00", "mtime_epoch": 1783315547.7880132, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209402}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:25:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1462.523, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.702, "pushed": 3.702, "read": 3.702}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.413, "count": 111, "max_ms": 5.453, "p95_ms": 4.851}, "gstreamer_push": {"avg_ms": 1.453, "count": 111, "max_ms": 4.285, "p95_ms": 3.323}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.506, "count": 111, "max_ms": 80.348, "p95_ms": 75.726}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.735, "count": 111, "max_ms": 113.705, "p95_ms": 98.562}, "yolo11_640_fire_11class": {"avg_ms": 58.924, "count": 111, "max_ms": 94.844, "p95_ms": 78.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.878, "count": 111, "max_ms": 97.965, "p95_ms": 94.789}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.23, "count": 111, "max_ms": 126.028, "p95_ms": 111.122}, "read_interval": {"avg_ms": 270.337, "count": 111, "max_ms": 10122.537, "p95_ms": 126.816}, "triton_wait": {"avg_ms": 72.182, "count": 111, "max_ms": 113.719, "p95_ms": 98.576}, "visualize": {"avg_ms": 3.658, "count": 111, "max_ms": 5.786, "p95_ms": 4.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 448}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5447.477, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:25:47+09:00", "last_error_summary": "[h264 @ 0x5584f31b37c0] left block unavailable for requested intra mode [h264 @ 0x5584f31b37c0] error while decoding MB 0 27, bytestream 48619 [h264 @ 0x5584f27b4000] reference picture missing during reorder [h264 @ 0x5584f27b4000] Missing reference picture, default is 65584 [h264 @ 0x5584f27b69c0] mmco: unref short failure [h264 @ 0x5584f27b69c0] illegal short term buffer state detected [h264 @ 0x5584f34ef1c0] top block unavailable for requested intra mode -1 [h264 @ 0x5584f34ef1c0] error while", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:25:47+09:00", "last_error_summary": "[h264 @ 0x5584f31b37c0] left block unavailable for requested intra mode [h264 @ 0x5584f31b37c0] error while decoding MB 0 27, bytestream 48619 [h264 @ 0x5584f27b4000] reference picture missing during reorder [h264 @ 0x5584f27b4000] Missing reference picture, default is 65584 [h264 @ 0x5584f27b69c0] mmco: unref short failure [h264 @ 0x5584f27b69c0] illegal short term buffer state detected [h264 @ 0x5584f34ef1c0] top block unavailable for requested intra mode -1 [h264 @ 0x5584f34ef1c0] error while", "last_exit_at": "2026-07-06T14:25:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 426.698, "last_started_at": "2026-07-06T14:18:41+09:00", "pid": 10367, "process": {"exists": false, "pid": 10367}, "restart_count": 12, "restart_count_per_hour": 8, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.273, "sample_count": 2671, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53792, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:25:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T14:25:47+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-06T14:25:47+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-06T14:25:47.788013+09:00", "mtime_epoch": 1783315547.7880132, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209402}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:25:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1462.523, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.701, "pushed": 3.701, "read": 3.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.4, "count": 111, "max_ms": 5.453, "p95_ms": 4.851}, "gstreamer_push": {"avg_ms": 1.447, "count": 111, "max_ms": 4.285, "p95_ms": 3.323}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.45, "count": 111, "max_ms": 80.348, "p95_ms": 75.726}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.703, "count": 111, "max_ms": 113.705, "p95_ms": 98.562}, "yolo11_640_fire_11class": {"avg_ms": 58.964, "count": 111, "max_ms": 94.844, "p95_ms": 78.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.811, "count": 111, "max_ms": 97.965, "p95_ms": 94.789}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.157, "count": 111, "max_ms": 126.028, "p95_ms": 111.122}, "read_interval": {"avg_ms": 270.337, "count": 111, "max_ms": 10122.537, "p95_ms": 126.816}, "triton_wait": {"avg_ms": 72.15, "count": 111, "max_ms": 113.719, "p95_ms": 98.576}, "visualize": {"avg_ms": 3.646, "count": 111, "max_ms": 5.786, "p95_ms": 4.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 448}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5447.465, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:25:47+09:00", "last_error_summary": "[h264 @ 0x5584f31b37c0] left block unavailable for requested intra mode [h264 @ 0x5584f31b37c0] error while decoding MB 0 27, bytestream 48619 [h264 @ 0x5584f27b4000] reference picture missing during reorder [h264 @ 0x5584f27b4000] Missing reference picture, default is 65584 [h264 @ 0x5584f27b69c0] mmco: unref short failure [h264 @ 0x5584f27b69c0] illegal short term buffer state detected [h264 @ 0x5584f34ef1c0] top block unavailable for requested intra mode -1 [h264 @ 0x5584f34ef1c0] error while", "last_exit_at": "2026-07-06T14:25:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 426.704, "last_started_at": "2026-07-06T14:18:41+09:00", "pid": 10367, "process": {"exists": false, "pid": 10367}, "restart_count": 12, "restart_count_per_hour": 8, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.273, "sample_count": 2671, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53792, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:25:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T14:25:47+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-06T14:25:47+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-06T14:25:47.788013+09:00", "mtime_epoch": 1783315547.7880132, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209402}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:25:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1462.523, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.7, "pushed": 3.7, "read": 3.7}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.4, "count": 111, "max_ms": 5.453, "p95_ms": 4.851}, "gstreamer_push": {"avg_ms": 1.447, "count": 111, "max_ms": 4.285, "p95_ms": 3.323}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.45, "count": 111, "max_ms": 80.348, "p95_ms": 75.726}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.703, "count": 111, "max_ms": 113.705, "p95_ms": 98.562}, "yolo11_640_fire_11class": {"avg_ms": 58.964, "count": 111, "max_ms": 94.844, "p95_ms": 78.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.811, "count": 111, "max_ms": 97.965, "p95_ms": 94.789}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.157, "count": 111, "max_ms": 126.028, "p95_ms": 111.122}, "read_interval": {"avg_ms": 270.337, "count": 111, "max_ms": 10122.537, "p95_ms": 126.816}, "triton_wait": {"avg_ms": 72.15, "count": 111, "max_ms": 113.719, "p95_ms": 98.576}, "visualize": {"avg_ms": 3.646, "count": 111, "max_ms": 5.786, "p95_ms": 4.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 448}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5563.645, "limit_mb": 8796093022207.996}, "detail": {"pid": 10930, "restart_count": 13, "start_count": 14}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:25:47+09:00", "last_error_summary": "[h264 @ 0x5584f31b37c0] left block unavailable for requested intra mode [h264 @ 0x5584f31b37c0] error while decoding MB 0 27, bytestream 48619 [h264 @ 0x5584f27b4000] reference picture missing during reorder [h264 @ 0x5584f27b4000] Missing reference picture, default is 65584 [h264 @ 0x5584f27b69c0] mmco: unref short failure [h264 @ 0x5584f27b69c0] illegal short term buffer state detected [h264 @ 0x5584f34ef1c0] top block unavailable for requested intra mode -1 [h264 @ 0x5584f34ef1c0] error while", "last_exit_at": "2026-07-06T14:25:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:25:50+09:00", "pid": 10930, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.672, "name": "ffmpeg", "pid": 10930, "rss_mb": 136.672, "state": "S (sleeping)", "threads": 106}, "restart_count": 13, "restart_count_per_hour": 9, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.273, "sample_count": 2631, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53792, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:25:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.332, "last_frame_pushed_at": "2026-07-06T14:25:47+09:00", "last_frame_read_age_sec": 4.41, "last_frame_read_at": "2026-07-06T14:25:47+09:00", "mjpeg": {"age_sec": 4.316, "exists": true, "mtime": "2026-07-06T14:25:47.872013+09:00", "mtime_epoch": 1783315547.872013, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209256}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:25:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1462.523, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.27, "pushed": 2.27, "read": 2.237}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.437, "count": 68, "max_ms": 5.453, "p95_ms": 4.855}, "gstreamer_push": {"avg_ms": 1.403, "count": 68, "max_ms": 3.933, "p95_ms": 1.697}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.939, "count": 68, "max_ms": 75.726, "p95_ms": 63.698}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.885, "count": 68, "max_ms": 105.655, "p95_ms": 94.584}, "yolo11_640_fire_11class": {"avg_ms": 58.722, "count": 68, "max_ms": 94.844, "p95_ms": 78.792}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.752, "count": 68, "max_ms": 97.965, "p95_ms": 94.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.404, "count": 68, "max_ms": 117.677, "p95_ms": 106.81}, "read_interval": {"avg_ms": 382.849, "count": 67, "max_ms": 10122.537, "p95_ms": 126.615}, "triton_wait": {"avg_ms": 71.442, "count": 68, "max_ms": 105.667, "p95_ms": 94.758}, "visualize": {"avg_ms": 3.628, "count": 68, "max_ms": 4.615, "p95_ms": 4.316}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 391}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5389.582, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a490a5cd80] error while decoding MB 116 30, bytestream -19 [in#0/rtsp @ 0x55a48fef2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a48fef2e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:26:48+09:00", "last_error_summary": "[h264 @ 0x55a490a5cd80] error while decoding MB 116 30, bytestream -19 [in#0/rtsp @ 0x55a48fef2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a48fef2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:25:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.606, "last_started_at": "2026-07-06T14:25:08+09:00", "pid": 10778, "process": {"exists": false, "pid": 10778}, "restart_count": 18, "restart_count_per_hour": 9, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.702, "sample_count": 1545, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47024, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:26:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.082, "last_frame_pushed_at": "2026-07-06T14:26:48+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-06T14:26:48+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-06T14:26:48.919888+09:00", "mtime_epoch": 1783315608.9198883, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:26:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.016, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.707, "pushed": 0.707, "read": 0.762}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.2, "count": 14, "max_ms": 6.486, "p95_ms": 6.486}, "gstreamer_push": {"avg_ms": 1.322, "count": 13, "max_ms": 2.571, "p95_ms": 2.571}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.203, "count": 14, "max_ms": 80.051, "p95_ms": 80.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.108, "count": 14, "max_ms": 100.767, "p95_ms": 100.767}, "yolo11_640_fire_11class": {"avg_ms": 60.314, "count": 14, "max_ms": 79.531, "p95_ms": 79.531}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.832, "count": 14, "max_ms": 93.78, "p95_ms": 93.78}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.447, "count": 13, "max_ms": 156.689, "p95_ms": 156.689}, "read_interval": {"avg_ms": 2436.558, "count": 14, "max_ms": 16777.697, "p95_ms": 16777.697}, "triton_wait": {"avg_ms": 76.935, "count": 14, "max_ms": 100.779, "p95_ms": 100.779}, "visualize": {"avg_ms": 4.523, "count": 14, "max_ms": 5.347, "p95_ms": 5.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.382, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 391}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5380.215, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:26:48+09:00", "last_error_summary": "[h264 @ 0x55a490a5cd80] error while decoding MB 116 30, bytestream -19 [in#0/rtsp @ 0x55a48fef2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a48fef2e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:26:48+09:00", "last_error_summary": "[h264 @ 0x55a490a5cd80] error while decoding MB 116 30, bytestream -19 [in#0/rtsp @ 0x55a48fef2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a48fef2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:26:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.657, "last_started_at": "2026-07-06T14:25:08+09:00", "pid": 10778, "process": {"exists": false, "pid": 10778}, "restart_count": 18, "restart_count_per_hour": 9, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.702, "sample_count": 1545, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47024, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:26:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T14:26:49+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T14:26:48+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T14:26:48.919888+09:00", "mtime_epoch": 1783315608.9198883, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:26:49+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.016, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.76, "pushed": 0.76, "read": 0.76}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.2, "count": 14, "max_ms": 6.486, "p95_ms": 6.486}, "gstreamer_push": {"avg_ms": 1.312, "count": 14, "max_ms": 2.571, "p95_ms": 2.571}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.203, "count": 14, "max_ms": 80.051, "p95_ms": 80.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.108, "count": 14, "max_ms": 100.767, "p95_ms": 100.767}, "yolo11_640_fire_11class": {"avg_ms": 60.314, "count": 14, "max_ms": 79.531, "p95_ms": 79.531}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.832, "count": 14, "max_ms": 93.78, "p95_ms": 93.78}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.878, "count": 14, "max_ms": 156.689, "p95_ms": 156.689}, "read_interval": {"avg_ms": 2436.558, "count": 14, "max_ms": 16777.697, "p95_ms": 16777.697}, "triton_wait": {"avg_ms": 76.935, "count": 14, "max_ms": 100.779, "p95_ms": 100.779}, "visualize": {"avg_ms": 4.523, "count": 14, "max_ms": 5.347, "p95_ms": 5.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.433, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 391}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5380.207, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:26:48+09:00", "last_error_summary": "[h264 @ 0x55a490a5cd80] error while decoding MB 116 30, bytestream -19 [in#0/rtsp @ 0x55a48fef2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a48fef2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:26:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.663, "last_started_at": "2026-07-06T14:25:08+09:00", "pid": 10778, "process": {"exists": false, "pid": 10778}, "restart_count": 18, "restart_count_per_hour": 9, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.702, "sample_count": 1545, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47024, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:26:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T14:26:49+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T14:26:48+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T14:26:48.919888+09:00", "mtime_epoch": 1783315608.9198883, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:26:49+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.016, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.759, "pushed": 0.759, "read": 0.759}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.2, "count": 14, "max_ms": 6.486, "p95_ms": 6.486}, "gstreamer_push": {"avg_ms": 1.312, "count": 14, "max_ms": 2.571, "p95_ms": 2.571}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.203, "count": 14, "max_ms": 80.051, "p95_ms": 80.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.108, "count": 14, "max_ms": 100.767, "p95_ms": 100.767}, "yolo11_640_fire_11class": {"avg_ms": 60.314, "count": 14, "max_ms": 79.531, "p95_ms": 79.531}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.832, "count": 14, "max_ms": 93.78, "p95_ms": 93.78}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.878, "count": 14, "max_ms": 156.689, "p95_ms": 156.689}, "read_interval": {"avg_ms": 2436.558, "count": 14, "max_ms": 16777.697, "p95_ms": 16777.697}, "triton_wait": {"avg_ms": 76.935, "count": 14, "max_ms": 100.779, "p95_ms": 100.779}, "visualize": {"avg_ms": 4.523, "count": 14, "max_ms": 5.347, "p95_ms": 5.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.439, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 391}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5563.762, "limit_mb": 8796093022207.996}, "detail": {"pid": 11037, "restart_count": 19, "start_count": 20}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:26:48+09:00", "last_error_summary": "[h264 @ 0x55a490a5cd80] error while decoding MB 116 30, bytestream -19 [in#0/rtsp @ 0x55a48fef2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a48fef2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:26:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:26:52+09:00", "pid": 11037, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.195, "name": "ffmpeg", "pid": 11037, "rss_mb": 204.195, "state": "S (sleeping)", "threads": 106}, "restart_count": 19, "restart_count_per_hour": 10, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.461, "sample_count": 1502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47024, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:26:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.994, "last_frame_pushed_at": "2026-07-06T14:26:49+09:00", "last_frame_read_age_sec": 5.118, "last_frame_read_at": "2026-07-06T14:26:48+09:00", "mjpeg": {"age_sec": 4.973, "exists": true, "mtime": "2026-07-06T14:26:49.051888+09:00", "mtime_epoch": 1783315609.051888, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221592}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:26:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.016, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.598, "pushed": 0.598, "read": 0.598}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.2, "count": 14, "max_ms": 6.486, "p95_ms": 6.486}, "gstreamer_push": {"avg_ms": 1.312, "count": 14, "max_ms": 2.571, "p95_ms": 2.571}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.203, "count": 14, "max_ms": 80.051, "p95_ms": 80.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.108, "count": 14, "max_ms": 100.767, "p95_ms": 100.767}, "yolo11_640_fire_11class": {"avg_ms": 60.314, "count": 14, "max_ms": 79.531, "p95_ms": 79.531}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.832, "count": 14, "max_ms": 93.78, "p95_ms": 93.78}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.878, "count": 14, "max_ms": 156.689, "p95_ms": 156.689}, "read_interval": {"avg_ms": 2436.558, "count": 14, "max_ms": 16777.697, "p95_ms": 16777.697}, "triton_wait": {"avg_ms": 76.935, "count": 14, "max_ms": 100.779, "p95_ms": 100.779}, "visualize": {"avg_ms": 4.523, "count": 14, "max_ms": 5.347, "p95_ms": 5.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.423, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 397}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5387.859, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5638f308e640] error while decoding MB 93 35, bytestream -5 [in#0/rtsp @ 0x5638f3009e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5638f3009e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:28:48+09:00", "last_error_summary": "[h264 @ 0x5638f308e640] error while decoding MB 93 35, bytestream -5 [in#0/rtsp @ 0x5638f3009e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5638f3009e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:26:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.799, "last_started_at": "2026-07-06T14:26:52+09:00", "pid": 11037, "process": {"exists": false, "pid": 11037}, "restart_count": 19, "restart_count_per_hour": 10, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.223, "sample_count": 1612, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47730, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:28:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-06T14:28:48+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T14:28:48+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-06T14:28:48.760029+09:00", "mtime_epoch": 1783315728.7600286, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:28:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1321.676, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.699, "pushed": 0.699, "read": 0.758}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.319, "count": 12, "max_ms": 6.803, "p95_ms": 6.803}, "gstreamer_push": {"avg_ms": 2.109, "count": 12, "max_ms": 4.659, "p95_ms": 4.659}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.998, "count": 12, "max_ms": 75.168, "p95_ms": 75.168}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.227, "count": 12, "max_ms": 101.182, "p95_ms": 101.182}, "yolo11_640_fire_11class": {"avg_ms": 61.62, "count": 12, "max_ms": 76.579, "p95_ms": 76.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.132, "count": 12, "max_ms": 93.025, "p95_ms": 93.025}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.089, "count": 12, "max_ms": 149.513, "p95_ms": 149.513}, "read_interval": {"avg_ms": 2459.258, "count": 13, "max_ms": 15918.733, "p95_ms": 15918.733}, "triton_wait": {"avg_ms": 76.446, "count": 12, "max_ms": 101.195, "p95_ms": 101.195}, "visualize": {"avg_ms": 4.02, "count": 12, "max_ms": 5.804, "p95_ms": 5.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.159, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 397}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5387.395, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:28:48+09:00", "last_error_summary": "[h264 @ 0x5638f308e640] error while decoding MB 93 35, bytestream -5 [in#0/rtsp @ 0x5638f3009e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5638f3009e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:28:48+09:00", "last_error_summary": "[h264 @ 0x5638f308e640] error while decoding MB 93 35, bytestream -5 [in#0/rtsp @ 0x5638f3009e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5638f3009e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:28:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.838, "last_started_at": "2026-07-06T14:26:52+09:00", "pid": 11037, "process": {"exists": false, "pid": 11037}, "restart_count": 19, "restart_count_per_hour": 10, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.223, "sample_count": 1613, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47731, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:28:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T14:28:48+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-06T14:28:48+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T14:28:48.760029+09:00", "mtime_epoch": 1783315728.7600286, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:28:48+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1321.676, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.756, "pushed": 0.756, "read": 0.756}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.289, "count": 13, "max_ms": 6.803, "p95_ms": 6.803}, "gstreamer_push": {"avg_ms": 2.07, "count": 13, "max_ms": 4.659, "p95_ms": 4.659}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.659, "count": 13, "max_ms": 75.168, "p95_ms": 75.168}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.794, "count": 13, "max_ms": 101.182, "p95_ms": 101.182}, "yolo11_640_fire_11class": {"avg_ms": 62.257, "count": 13, "max_ms": 76.579, "p95_ms": 76.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.743, "count": 13, "max_ms": 93.025, "p95_ms": 93.025}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.094, "count": 13, "max_ms": 149.513, "p95_ms": 149.513}, "read_interval": {"avg_ms": 2459.258, "count": 13, "max_ms": 15918.733, "p95_ms": 15918.733}, "triton_wait": {"avg_ms": 76.997, "count": 13, "max_ms": 101.195, "p95_ms": 101.195}, "visualize": {"avg_ms": 3.995, "count": 13, "max_ms": 5.804, "p95_ms": 5.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.198, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 397}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5387.387, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:28:48+09:00", "last_error_summary": "[h264 @ 0x5638f308e640] error while decoding MB 93 35, bytestream -5 [in#0/rtsp @ 0x5638f3009e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5638f3009e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:28:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.845, "last_started_at": "2026-07-06T14:26:52+09:00", "pid": 11037, "process": {"exists": false, "pid": 11037}, "restart_count": 19, "restart_count_per_hour": 10, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.223, "sample_count": 1613, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47731, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:28:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T14:28:48+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T14:28:48+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T14:28:48.760029+09:00", "mtime_epoch": 1783315728.7600286, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:28:48+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1321.676, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.756, "pushed": 0.756, "read": 0.756}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.289, "count": 13, "max_ms": 6.803, "p95_ms": 6.803}, "gstreamer_push": {"avg_ms": 2.07, "count": 13, "max_ms": 4.659, "p95_ms": 4.659}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.659, "count": 13, "max_ms": 75.168, "p95_ms": 75.168}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.794, "count": 13, "max_ms": 101.182, "p95_ms": 101.182}, "yolo11_640_fire_11class": {"avg_ms": 62.257, "count": 13, "max_ms": 76.579, "p95_ms": 76.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.743, "count": 13, "max_ms": 93.025, "p95_ms": 93.025}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.094, "count": 13, "max_ms": 149.513, "p95_ms": 149.513}, "read_interval": {"avg_ms": 2459.258, "count": 13, "max_ms": 15918.733, "p95_ms": 15918.733}, "triton_wait": {"avg_ms": 76.997, "count": 13, "max_ms": 101.195, "p95_ms": 101.195}, "visualize": {"avg_ms": 3.995, "count": 13, "max_ms": 5.804, "p95_ms": 5.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.204, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 397}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5515.438, "limit_mb": 8796093022207.996}, "detail": {"pid": 11189, "restart_count": 20, "start_count": 21}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:28:48+09:00", "last_error_summary": "[h264 @ 0x5638f308e640] error while decoding MB 93 35, bytestream -5 [in#0/rtsp @ 0x5638f3009e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5638f3009e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:28:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:28:51+09:00", "pid": 11189, "process": {"exists": true, "fd_count": 5, "hwm_mb": 157.098, "name": "ffmpeg", "pid": 11189, "rss_mb": 156.934, "state": "R (running)", "threads": 49}, "restart_count": 20, "restart_count_per_hour": 11, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.452, "sample_count": 1570, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47731, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:28:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 5.006, "last_frame_pushed_at": "2026-07-06T14:28:48+09:00", "last_frame_read_age_sec": 5.105, "last_frame_read_at": "2026-07-06T14:28:48+09:00", "mjpeg": {"age_sec": 4.99, "exists": true, "mtime": "2026-07-06T14:28:48.872029+09:00", "mtime_epoch": 1783315728.872029, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231616}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:28:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1321.676, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.586, "pushed": 0.586, "read": 0.586}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.289, "count": 13, "max_ms": 6.803, "p95_ms": 6.803}, "gstreamer_push": {"avg_ms": 2.07, "count": 13, "max_ms": 4.659, "p95_ms": 4.659}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.659, "count": 13, "max_ms": 75.168, "p95_ms": 75.168}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.794, "count": 13, "max_ms": 101.182, "p95_ms": 101.182}, "yolo11_640_fire_11class": {"avg_ms": 62.257, "count": 13, "max_ms": 76.579, "p95_ms": 76.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.743, "count": 13, "max_ms": 93.025, "p95_ms": 93.025}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.094, "count": 13, "max_ms": 149.513, "p95_ms": 149.513}, "read_interval": {"avg_ms": 2459.258, "count": 13, "max_ms": 15918.733, "p95_ms": 15918.733}, "triton_wait": {"avg_ms": 76.997, "count": 13, "max_ms": 101.195, "p95_ms": 101.195}, "visualize": {"avg_ms": 3.995, "count": 13, "max_ms": 5.804, "p95_ms": 5.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.198, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 474}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5456.586, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d8af27c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8af27c600] error while decoding MB 0 30, bytestream 87291 [h264 @ 0x55d8af327c40] left block unavailable for requested intra mode [h264 @ 0x55d8af327c40] error while decoding MB 0 15, bytestream 2876 [h264 @ 0x55d8af226380] cabac decode of qscale diff failed at 22 4 [h264 @ 0x55d8af226380] error while decoding MB 22 4, bytestream 106 [h264 @ 0x55d8af364000] cabac decode of qscale diff failed at 50 44 [h264", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 68, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:31:49+09:00", "last_error_summary": "[h264 @ 0x55d8af27c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8af27c600] error while decoding MB 0 30, bytestream 87291 [h264 @ 0x55d8af327c40] left block unavailable for requested intra mode [h264 @ 0x55d8af327c40] error while decoding MB 0 15, bytestream 2876 [h264 @ 0x55d8af226380] cabac decode of qscale diff failed at 22 4 [h264 @ 0x55d8af226380] error while decoding MB 22 4, bytestream 106 [h264 @ 0x55d8af364000] cabac decode of qscale diff failed at 50 44 [h264", "last_exit_at": "2026-07-06T14:25:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 359.147, "last_started_at": "2026-07-06T14:25:50+09:00", "pid": 10930, "process": {"exists": false, "pid": 10930}, "restart_count": 13, "restart_count_per_hour": 8, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.276, "sample_count": 2531, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56886, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:31:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T14:31:49+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T14:31:49+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-06T14:31:49.281006+09:00", "mtime_epoch": 1783315909.2810056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191964}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:31:49+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1464.277, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.505, "pushed": 3.505, "read": 3.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.701, "count": 105, "max_ms": 6.146, "p95_ms": 5.65}, "gstreamer_push": {"avg_ms": 1.453, "count": 105, "max_ms": 4.5, "p95_ms": 3.257}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.032, "count": 105, "max_ms": 81.836, "p95_ms": 75.924}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.611, "count": 105, "max_ms": 125.976, "p95_ms": 96.675}, "yolo11_640_fire_11class": {"avg_ms": 58.918, "count": 105, "max_ms": 80.979, "p95_ms": 79.055}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.582, "count": 105, "max_ms": 99.092, "p95_ms": 92.888}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.597, "count": 105, "max_ms": 153.006, "p95_ms": 140.743}, "read_interval": {"avg_ms": 285.289, "count": 105, "max_ms": 19292.691, "p95_ms": 155.728}, "triton_wait": {"avg_ms": 71.917, "count": 105, "max_ms": 125.988, "p95_ms": 96.689}, "visualize": {"avg_ms": 3.897, "count": 105, "max_ms": 5.398, "p95_ms": 5.155}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 474}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5455.805, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:31:49+09:00", "last_error_summary": "[h264 @ 0x55d8af27c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8af27c600] error while decoding MB 0 30, bytestream 87291 [h264 @ 0x55d8af327c40] left block unavailable for requested intra mode [h264 @ 0x55d8af327c40] error while decoding MB 0 15, bytestream 2876 [h264 @ 0x55d8af226380] cabac decode of qscale diff failed at 22 4 [h264 @ 0x55d8af226380] error while decoding MB 22 4, bytestream 106 [h264 @ 0x55d8af364000] cabac decode of qscale diff failed at 50 44 [h264", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 68, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:31:49+09:00", "last_error_summary": "[h264 @ 0x55d8af27c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8af27c600] error while decoding MB 0 30, bytestream 87291 [h264 @ 0x55d8af327c40] left block unavailable for requested intra mode [h264 @ 0x55d8af327c40] error while decoding MB 0 15, bytestream 2876 [h264 @ 0x55d8af226380] cabac decode of qscale diff failed at 22 4 [h264 @ 0x55d8af226380] error while decoding MB 22 4, bytestream 106 [h264 @ 0x55d8af364000] cabac decode of qscale diff failed at 50 44 [h264", "last_exit_at": "2026-07-06T14:31:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 359.197, "last_started_at": "2026-07-06T14:25:50+09:00", "pid": 10930, "process": {"exists": false, "pid": 10930}, "restart_count": 13, "restart_count_per_hour": 8, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.226, "sample_count": 2532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:31:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T14:31:49+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-06T14:31:49+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-06T14:31:49.281006+09:00", "mtime_epoch": 1783315909.2810056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191964}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:31:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1464.277, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.501, "pushed": 3.501, "read": 3.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.695, "count": 105, "max_ms": 6.146, "p95_ms": 5.65}, "gstreamer_push": {"avg_ms": 1.455, "count": 105, "max_ms": 4.5, "p95_ms": 3.257}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.368, "count": 105, "max_ms": 81.836, "p95_ms": 75.924}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.006, "count": 105, "max_ms": 125.976, "p95_ms": 96.675}, "yolo11_640_fire_11class": {"avg_ms": 59.208, "count": 105, "max_ms": 80.979, "p95_ms": 79.055}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.944, "count": 105, "max_ms": 99.092, "p95_ms": 92.888}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.983, "count": 105, "max_ms": 153.006, "p95_ms": 140.743}, "read_interval": {"avg_ms": 285.289, "count": 105, "max_ms": 19292.691, "p95_ms": 155.728}, "triton_wait": {"avg_ms": 72.312, "count": 105, "max_ms": 125.988, "p95_ms": 96.689}, "visualize": {"avg_ms": 3.896, "count": 105, "max_ms": 5.398, "p95_ms": 5.155}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 474}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5455.918, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 68, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:31:49+09:00", "last_error_summary": "[h264 @ 0x55d8af27c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8af27c600] error while decoding MB 0 30, bytestream 87291 [h264 @ 0x55d8af327c40] left block unavailable for requested intra mode [h264 @ 0x55d8af327c40] error while decoding MB 0 15, bytestream 2876 [h264 @ 0x55d8af226380] cabac decode of qscale diff failed at 22 4 [h264 @ 0x55d8af226380] error while decoding MB 22 4, bytestream 106 [h264 @ 0x55d8af364000] cabac decode of qscale diff failed at 50 44 [h264", "last_exit_at": "2026-07-06T14:31:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 359.206, "last_started_at": "2026-07-06T14:25:50+09:00", "pid": 10930, "process": {"exists": false, "pid": 10930}, "restart_count": 13, "restart_count_per_hour": 8, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.226, "sample_count": 2532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:31:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T14:31:49+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-06T14:31:49+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T14:31:49.281006+09:00", "mtime_epoch": 1783315909.2810056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191964}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:31:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1464.277, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.511, "pushed": 3.511, "read": 3.478}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.695, "count": 105, "max_ms": 6.146, "p95_ms": 5.65}, "gstreamer_push": {"avg_ms": 1.455, "count": 105, "max_ms": 4.5, "p95_ms": 3.257}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.368, "count": 105, "max_ms": 81.836, "p95_ms": 75.924}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.006, "count": 105, "max_ms": 125.976, "p95_ms": 96.675}, "yolo11_640_fire_11class": {"avg_ms": 59.208, "count": 105, "max_ms": 80.979, "p95_ms": 79.055}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.944, "count": 105, "max_ms": 99.092, "p95_ms": 92.888}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.983, "count": 105, "max_ms": 153.006, "p95_ms": 140.743}, "read_interval": {"avg_ms": 287.291, "count": 104, "max_ms": 19292.691, "p95_ms": 155.728}, "triton_wait": {"avg_ms": 72.312, "count": 105, "max_ms": 125.988, "p95_ms": 96.689}, "visualize": {"avg_ms": 3.896, "count": 105, "max_ms": 5.398, "p95_ms": 5.155}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 474}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5565.703, "limit_mb": 8796093022207.996}, "detail": {"pid": 11341, "restart_count": 14, "start_count": 15}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 68, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:31:49+09:00", "last_error_summary": "[h264 @ 0x55d8af27c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8af27c600] error while decoding MB 0 30, bytestream 87291 [h264 @ 0x55d8af327c40] left block unavailable for requested intra mode [h264 @ 0x55d8af327c40] error while decoding MB 0 15, bytestream 2876 [h264 @ 0x55d8af226380] cabac decode of qscale diff failed at 22 4 [h264 @ 0x55d8af226380] error while decoding MB 22 4, bytestream 106 [h264 @ 0x55d8af364000] cabac decode of qscale diff failed at 50 44 [h264", "last_exit_at": "2026-07-06T14:31:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:31:52+09:00", "pid": 11341, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.262, "name": "ffmpeg", "pid": 11341, "rss_mb": 139.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 14, "restart_count_per_hour": 9, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.624, "sample_count": 2480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:31:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.958, "last_frame_pushed_at": "2026-07-06T14:31:49+09:00", "last_frame_read_age_sec": 5.065, "last_frame_read_at": "2026-07-06T14:31:49+09:00", "mjpeg": {"age_sec": 4.942, "exists": true, "mtime": "2026-07-06T14:31:49.393006+09:00", "mtime_epoch": 1783315909.3930063, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189742}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:31:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1464.277, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 53, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.806, "pushed": 1.806, "read": 1.772}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.981, "count": 54, "max_ms": 6.146, "p95_ms": 5.856}, "gstreamer_push": {"avg_ms": 1.471, "count": 54, "max_ms": 4.5, "p95_ms": 4.058}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.24, "count": 54, "max_ms": 80.357, "p95_ms": 75.684}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.676, "count": 54, "max_ms": 101.101, "p95_ms": 96.675}, "yolo11_640_fire_11class": {"avg_ms": 58.78, "count": 54, "max_ms": 80.925, "p95_ms": 79.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.312, "count": 54, "max_ms": 99.092, "p95_ms": 92.582}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.788, "count": 54, "max_ms": 153.006, "p95_ms": 148.055}, "read_interval": {"avg_ms": 470.487, "count": 53, "max_ms": 19292.691, "p95_ms": 162.583}, "triton_wait": {"avg_ms": 72.975, "count": 54, "max_ms": 101.114, "p95_ms": 96.689}, "visualize": {"avg_ms": 4.081, "count": 54, "max_ms": 5.398, "p95_ms": 5.376}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 410}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5383.527, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55aebe5a8680] error while decoding MB 60 22, bytestream -15 [in#0/rtsp @ 0x55aebd7c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aebd7c8e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:32:23+09:00", "last_error_summary": "[h264 @ 0x55aebe5a8680] error while decoding MB 60 22, bytestream -15 [in#0/rtsp @ 0x55aebd7c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aebd7c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:28:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 211.757, "last_started_at": "2026-07-06T14:28:51+09:00", "pid": 11189, "process": {"exists": false, "pid": 11189}, "restart_count": 20, "restart_count_per_hour": 9, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.627, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49230, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:32:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T14:32:23+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T14:32:23+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T14:32:23.561274+09:00", "mtime_epoch": 1783315943.5612743, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226725}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:32:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.562, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 31, "pushed_frames": 31, "read_frames": 31, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.037, "pushed": 1.037, "read": 1.037}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.829, "count": 31, "max_ms": 7.276, "p95_ms": 6.938}, "gstreamer_push": {"avg_ms": 1.165, "count": 31, "max_ms": 1.412, "p95_ms": 1.394}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.595, "count": 31, "max_ms": 76.832, "p95_ms": 73.35}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.749, "count": 31, "max_ms": 101.044, "p95_ms": 96.589}, "yolo11_640_fire_11class": {"avg_ms": 62.896, "count": 31, "max_ms": 84.574, "p95_ms": 79.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.882, "count": 31, "max_ms": 100.316, "p95_ms": 93.32}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.752, "count": 31, "max_ms": 148.091, "p95_ms": 147.604}, "read_interval": {"avg_ms": 966.587, "count": 31, "max_ms": 16039.984, "p95_ms": 10165.348}, "triton_wait": {"avg_ms": 76.628, "count": 31, "max_ms": 101.056, "p95_ms": 96.602}, "visualize": {"avg_ms": 4.825, "count": 31, "max_ms": 5.751, "p95_ms": 5.617}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 410}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5382.602, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:32:23+09:00", "last_error_summary": "[h264 @ 0x55aebe5a8680] error while decoding MB 60 22, bytestream -15 [in#0/rtsp @ 0x55aebd7c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aebd7c8e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:32:23+09:00", "last_error_summary": "[h264 @ 0x55aebe5a8680] error while decoding MB 60 22, bytestream -15 [in#0/rtsp @ 0x55aebd7c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aebd7c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:32:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 211.804, "last_started_at": "2026-07-06T14:28:51+09:00", "pid": 11189, "process": {"exists": false, "pid": 11189}, "restart_count": 20, "restart_count_per_hour": 9, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.627, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49231, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:32:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T14:32:23+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-06T14:32:23+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-06T14:32:23.561274+09:00", "mtime_epoch": 1783315943.5612743, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226725}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:32:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.562, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 32, "pushed_frames": 32, "read_frames": 31, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.069, "pushed": 1.069, "read": 1.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.804, "count": 32, "max_ms": 7.276, "p95_ms": 6.938}, "gstreamer_push": {"avg_ms": 1.167, "count": 32, "max_ms": 1.412, "p95_ms": 1.394}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.656, "count": 32, "max_ms": 76.832, "p95_ms": 73.35}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.239, "count": 32, "max_ms": 101.044, "p95_ms": 96.589}, "yolo11_640_fire_11class": {"avg_ms": 63.244, "count": 32, "max_ms": 84.574, "p95_ms": 79.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.314, "count": 32, "max_ms": 100.316, "p95_ms": 93.32}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.29, "count": 32, "max_ms": 148.091, "p95_ms": 147.604}, "read_interval": {"avg_ms": 966.587, "count": 31, "max_ms": 16039.984, "p95_ms": 10165.348}, "triton_wait": {"avg_ms": 77.091, "count": 32, "max_ms": 101.056, "p95_ms": 96.602}, "visualize": {"avg_ms": 4.786, "count": 32, "max_ms": 5.751, "p95_ms": 5.617}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 410}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5382.594, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:32:23+09:00", "last_error_summary": "[h264 @ 0x55aebe5a8680] error while decoding MB 60 22, bytestream -15 [in#0/rtsp @ 0x55aebd7c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aebd7c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:32:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 211.809, "last_started_at": "2026-07-06T14:28:51+09:00", "pid": 11189, "process": {"exists": false, "pid": 11189}, "restart_count": 20, "restart_count_per_hour": 9, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.627, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49231, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:32:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T14:32:23+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-06T14:32:23+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T14:32:23.561274+09:00", "mtime_epoch": 1783315943.5612743, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226725}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:32:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.562, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 32, "pushed_frames": 32, "read_frames": 31, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.069, "pushed": 1.069, "read": 1.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.804, "count": 32, "max_ms": 7.276, "p95_ms": 6.938}, "gstreamer_push": {"avg_ms": 1.167, "count": 32, "max_ms": 1.412, "p95_ms": 1.394}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.656, "count": 32, "max_ms": 76.832, "p95_ms": 73.35}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.239, "count": 32, "max_ms": 101.044, "p95_ms": 96.589}, "yolo11_640_fire_11class": {"avg_ms": 63.244, "count": 32, "max_ms": 84.574, "p95_ms": 79.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.314, "count": 32, "max_ms": 100.316, "p95_ms": 93.32}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.29, "count": 32, "max_ms": 148.091, "p95_ms": 147.604}, "read_interval": {"avg_ms": 966.587, "count": 31, "max_ms": 16039.984, "p95_ms": 10165.348}, "triton_wait": {"avg_ms": 77.091, "count": 32, "max_ms": 101.056, "p95_ms": 96.602}, "visualize": {"avg_ms": 4.786, "count": 32, "max_ms": 5.751, "p95_ms": 5.617}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 410}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5556.812, "limit_mb": 8796093022207.996}, "detail": {"pid": 11448, "restart_count": 21, "start_count": 22}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:32:23+09:00", "last_error_summary": "[h264 @ 0x55aebe5a8680] error while decoding MB 60 22, bytestream -15 [in#0/rtsp @ 0x55aebd7c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aebd7c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:32:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:32:25+09:00", "pid": 11448, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.598, "name": "ffmpeg", "pid": 11448, "rss_mb": 197.598, "state": "S (sleeping)", "threads": 106}, "restart_count": 21, "restart_count_per_hour": 10, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.627, "sample_count": 1927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49231, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:32:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.334, "last_frame_pushed_at": "2026-07-06T14:32:23+09:00", "last_frame_read_age_sec": 4.441, "last_frame_read_at": "2026-07-06T14:32:23+09:00", "mjpeg": {"age_sec": 4.316, "exists": true, "mtime": "2026-07-06T14:32:23.677275+09:00", "mtime_epoch": 1783315943.6772752, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226153}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:32:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1328.562, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.815, "pushed": 0.815, "read": 0.815}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.198, "count": 13, "max_ms": 7.276, "p95_ms": 7.276}, "gstreamer_push": {"avg_ms": 1.199, "count": 13, "max_ms": 1.412, "p95_ms": 1.412}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.869, "count": 13, "max_ms": 73.35, "p95_ms": 73.35}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.323, "count": 13, "max_ms": 101.044, "p95_ms": 101.044}, "yolo11_640_fire_11class": {"avg_ms": 65.717, "count": 13, "max_ms": 84.574, "p95_ms": 84.574}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.666, "count": 13, "max_ms": 100.316, "p95_ms": 100.316}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.38, "count": 13, "max_ms": 148.091, "p95_ms": 148.091}, "read_interval": {"avg_ms": 2119.848, "count": 13, "max_ms": 16039.984, "p95_ms": 16039.984}, "triton_wait": {"avg_ms": 80.086, "count": 13, "max_ms": 101.056, "p95_ms": 101.056}, "visualize": {"avg_ms": 4.386, "count": 13, "max_ms": 5.617, "p95_ms": 5.617}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.959, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 497}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5455.121, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563a5709a5c0] left block unavailable for requested intra mode [h264 @ 0x563a5709a5c0] error while decoding MB 0 26, bytestream 4120 [h264 @ 0x563a56649140] cabac decode of qscale diff failed at 69 42 [h264 @ 0x563a56649140] error while decoding MB 69 42, bytestream 50063 [h264 @ 0x563a566a1c80] left block unavailable for requested intra mode [h264 @ 0x563a566a1c80] error while decoding MB 0 24, bytestream 289 [h264 @ 0x563a572382c0] left block unavailable for requested intra4x4 mode -1", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 64, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:37:15+09:00", "last_error_summary": "[h264 @ 0x563a5709a5c0] left block unavailable for requested intra mode [h264 @ 0x563a5709a5c0] error while decoding MB 0 26, bytestream 4120 [h264 @ 0x563a56649140] cabac decode of qscale diff failed at 69 42 [h264 @ 0x563a56649140] error while decoding MB 69 42, bytestream 50063 [h264 @ 0x563a566a1c80] left block unavailable for requested intra mode [h264 @ 0x563a566a1c80] error while decoding MB 0 24, bytestream 289 [h264 @ 0x563a572382c0] left block unavailable for requested intra4x4 mode -1", "last_exit_at": "2026-07-06T14:31:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 323.372, "last_started_at": "2026-07-06T14:31:52+09:00", "pid": 11341, "process": {"exists": false, "pid": 11341}, "restart_count": 14, "restart_count_per_hour": 8, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.972, "sample_count": 2576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:37:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.001, "last_frame_pushed_at": "2026-07-06T14:37:15+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-06T14:37:15+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-06T14:37:15.632344+09:00", "mtime_epoch": 1783316235.6323442, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208342}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:37:15+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1468.637, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.474, "pushed": 3.474, "read": 3.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.388, "count": 104, "max_ms": 5.717, "p95_ms": 4.541}, "gstreamer_push": {"avg_ms": 1.367, "count": 104, "max_ms": 3.725, "p95_ms": 1.694}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.8, "count": 104, "max_ms": 74.755, "p95_ms": 70.267}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.404, "count": 104, "max_ms": 110.087, "p95_ms": 98.632}, "yolo11_640_fire_11class": {"avg_ms": 57.903, "count": 104, "max_ms": 87.899, "p95_ms": 77.557}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.919, "count": 104, "max_ms": 108.332, "p95_ms": 89.276}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.114, "count": 104, "max_ms": 145.862, "p95_ms": 115.3}, "read_interval": {"avg_ms": 290.508, "count": 103, "max_ms": 20263.451, "p95_ms": 132.2}, "triton_wait": {"avg_ms": 69.694, "count": 104, "max_ms": 110.099, "p95_ms": 103.164}, "visualize": {"avg_ms": 3.668, "count": 104, "max_ms": 5.555, "p95_ms": 4.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 497}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5455.07, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:37:15+09:00", "last_error_summary": "[h264 @ 0x563a5709a5c0] left block unavailable for requested intra mode [h264 @ 0x563a5709a5c0] error while decoding MB 0 26, bytestream 4120 [h264 @ 0x563a56649140] cabac decode of qscale diff failed at 69 42 [h264 @ 0x563a56649140] error while decoding MB 69 42, bytestream 50063 [h264 @ 0x563a566a1c80] left block unavailable for requested intra mode [h264 @ 0x563a566a1c80] error while decoding MB 0 24, bytestream 289 [h264 @ 0x563a572382c0] left block unavailable for requested intra4x4 mode -1", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 64, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:37:15+09:00", "last_error_summary": "[h264 @ 0x563a5709a5c0] left block unavailable for requested intra mode [h264 @ 0x563a5709a5c0] error while decoding MB 0 26, bytestream 4120 [h264 @ 0x563a56649140] cabac decode of qscale diff failed at 69 42 [h264 @ 0x563a56649140] error while decoding MB 69 42, bytestream 50063 [h264 @ 0x563a566a1c80] left block unavailable for requested intra mode [h264 @ 0x563a566a1c80] error while decoding MB 0 24, bytestream 289 [h264 @ 0x563a572382c0] left block unavailable for requested intra4x4 mode -1", "last_exit_at": "2026-07-06T14:37:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 323.386, "last_started_at": "2026-07-06T14:31:52+09:00", "pid": 11341, "process": {"exists": false, "pid": 11341}, "restart_count": 14, "restart_count_per_hour": 8, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.972, "sample_count": 2576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:37:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T14:37:15+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-06T14:37:15+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-06T14:37:15.632344+09:00", "mtime_epoch": 1783316235.6323442, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208342}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:37:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1468.637, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.473, "pushed": 3.473, "read": 3.439}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.388, "count": 104, "max_ms": 5.717, "p95_ms": 4.541}, "gstreamer_push": {"avg_ms": 1.367, "count": 104, "max_ms": 3.725, "p95_ms": 1.694}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.8, "count": 104, "max_ms": 74.755, "p95_ms": 70.267}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.404, "count": 104, "max_ms": 110.087, "p95_ms": 98.632}, "yolo11_640_fire_11class": {"avg_ms": 57.903, "count": 104, "max_ms": 87.899, "p95_ms": 77.557}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.919, "count": 104, "max_ms": 108.332, "p95_ms": 89.276}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.114, "count": 104, "max_ms": 145.862, "p95_ms": 115.3}, "read_interval": {"avg_ms": 290.508, "count": 103, "max_ms": 20263.451, "p95_ms": 132.2}, "triton_wait": {"avg_ms": 69.694, "count": 104, "max_ms": 110.099, "p95_ms": 103.164}, "visualize": {"avg_ms": 3.668, "count": 104, "max_ms": 5.555, "p95_ms": 4.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 497}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5454.707, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 64, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:37:15+09:00", "last_error_summary": "[h264 @ 0x563a5709a5c0] left block unavailable for requested intra mode [h264 @ 0x563a5709a5c0] error while decoding MB 0 26, bytestream 4120 [h264 @ 0x563a56649140] cabac decode of qscale diff failed at 69 42 [h264 @ 0x563a56649140] error while decoding MB 69 42, bytestream 50063 [h264 @ 0x563a566a1c80] left block unavailable for requested intra mode [h264 @ 0x563a566a1c80] error while decoding MB 0 24, bytestream 289 [h264 @ 0x563a572382c0] left block unavailable for requested intra4x4 mode -1", "last_exit_at": "2026-07-06T14:37:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 323.393, "last_started_at": "2026-07-06T14:31:52+09:00", "pid": 11341, "process": {"exists": false, "pid": 11341}, "restart_count": 14, "restart_count_per_hour": 8, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.972, "sample_count": 2576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:37:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-06T14:37:15+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-06T14:37:15+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-06T14:37:15.632344+09:00", "mtime_epoch": 1783316235.6323442, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208342}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:37:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1468.637, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.472, "pushed": 3.472, "read": 3.439}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.388, "count": 104, "max_ms": 5.717, "p95_ms": 4.541}, "gstreamer_push": {"avg_ms": 1.367, "count": 104, "max_ms": 3.725, "p95_ms": 1.694}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.8, "count": 104, "max_ms": 74.755, "p95_ms": 70.267}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.404, "count": 104, "max_ms": 110.087, "p95_ms": 98.632}, "yolo11_640_fire_11class": {"avg_ms": 57.903, "count": 104, "max_ms": 87.899, "p95_ms": 77.557}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.919, "count": 104, "max_ms": 108.332, "p95_ms": 89.276}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.114, "count": 104, "max_ms": 145.862, "p95_ms": 115.3}, "read_interval": {"avg_ms": 290.508, "count": 103, "max_ms": 20263.451, "p95_ms": 132.2}, "triton_wait": {"avg_ms": 69.694, "count": 104, "max_ms": 110.099, "p95_ms": 103.164}, "visualize": {"avg_ms": 3.668, "count": 104, "max_ms": 5.555, "p95_ms": 4.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 497}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5563.18, "limit_mb": 8796093022207.996}, "detail": {"pid": 11600, "restart_count": 15, "start_count": 16}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 64, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:37:15+09:00", "last_error_summary": "[h264 @ 0x563a5709a5c0] left block unavailable for requested intra mode [h264 @ 0x563a5709a5c0] error while decoding MB 0 26, bytestream 4120 [h264 @ 0x563a56649140] cabac decode of qscale diff failed at 69 42 [h264 @ 0x563a56649140] error while decoding MB 69 42, bytestream 50063 [h264 @ 0x563a566a1c80] left block unavailable for requested intra mode [h264 @ 0x563a566a1c80] error while decoding MB 0 24, bytestream 289 [h264 @ 0x563a572382c0] left block unavailable for requested intra4x4 mode -1", "last_exit_at": "2026-07-06T14:37:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:37:18+09:00", "pid": 11600, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.547, "name": "ffmpeg", "pid": 11600, "rss_mb": 136.547, "state": "S (sleeping)", "threads": 106}, "restart_count": 15, "restart_count_per_hour": 9, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.096, "sample_count": 2528, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:37:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.865, "last_frame_pushed_at": "2026-07-06T14:37:15+09:00", "last_frame_read_age_sec": 4.943, "last_frame_read_at": "2026-07-06T14:37:15+09:00", "mjpeg": {"age_sec": 4.85, "exists": true, "mtime": "2026-07-06T14:37:15.720345+09:00", "mtime_epoch": 1783316235.7203453, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208877}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:37:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1468.637, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 53, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.4, "pushed": 5.4, "read": 5.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.372, "count": 53, "max_ms": 5.717, "p95_ms": 4.541}, "gstreamer_push": {"avg_ms": 1.336, "count": 53, "max_ms": 2.118, "p95_ms": 1.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.84, "count": 53, "max_ms": 73.312, "p95_ms": 70.267}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.094, "count": 53, "max_ms": 109.165, "p95_ms": 97.988}, "yolo11_640_fire_11class": {"avg_ms": 58.566, "count": 53, "max_ms": 87.899, "p95_ms": 79.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.211, "count": 53, "max_ms": 96.386, "p95_ms": 83.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.994, "count": 53, "max_ms": 145.862, "p95_ms": 120.325}, "read_interval": {"avg_ms": 474.254, "count": 53, "max_ms": 20263.451, "p95_ms": 132.52}, "triton_wait": {"avg_ms": 69.259, "count": 53, "max_ms": 109.18, "p95_ms": 98.0}, "visualize": {"avg_ms": 3.68, "count": 53, "max_ms": 5.555, "p95_ms": 4.482}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 9.815, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 540}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5461.141, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 12, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55887f9342c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f9342c0] error while decoding MB 0 13, bytestream 4364 [h264 @ 0x55887f9342c0] cabac decode of qscale diff failed at 29 32 [h264 @ 0x55887f9342c0] error while decoding MB 29 32, bytestream 6152 [h264 @ 0x55887f7965c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f7965c0] error while decoding MB 0 3, bytestream 3754 [h264 @ 0x55887ecb9580] left block unavailable for requested intra", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 104, "h264_decode_error_count_per_hour": 64, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:46:51+09:00", "last_error_summary": "[h264 @ 0x55887f9342c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f9342c0] error while decoding MB 0 13, bytestream 4364 [h264 @ 0x55887f9342c0] cabac decode of qscale diff failed at 29 32 [h264 @ 0x55887f9342c0] error while decoding MB 29 32, bytestream 6152 [h264 @ 0x55887f7965c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f7965c0] error while decoding MB 0 3, bytestream 3754 [h264 @ 0x55887ecb9580] left block unavailable for requested intra", "last_exit_at": "2026-07-06T14:37:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 572.829, "last_started_at": "2026-07-06T14:37:18+09:00", "pid": 11600, "process": {"exists": false, "pid": 11600}, "restart_count": 15, "restart_count_per_hour": 7, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 132.006, "sample_count": 2647, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64914, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:46:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T14:46:51+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T14:46:51+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T14:46:51.348780+09:00", "mtime_epoch": 1783316811.3487802, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:46:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1467.84, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 81, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.702, "pushed": 2.702, "read": 2.702}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.423, "count": 80, "max_ms": 8.205, "p95_ms": 7.308}, "gstreamer_push": {"avg_ms": 1.252, "count": 81, "max_ms": 2.866, "p95_ms": 1.495}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.386, "count": 80, "max_ms": 81.181, "p95_ms": 76.652}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.6, "count": 80, "max_ms": 116.782, "p95_ms": 102.275}, "yolo11_640_fire_11class": {"avg_ms": 61.102, "count": 80, "max_ms": 85.548, "p95_ms": 80.305}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.305, "count": 80, "max_ms": 103.772, "p95_ms": 97.898}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.722, "count": 81, "max_ms": 169.659, "p95_ms": 159.697}, "read_interval": {"avg_ms": 371.251, "count": 81, "max_ms": 19591.331, "p95_ms": 169.975}, "triton_wait": {"avg_ms": 76.248, "count": 80, "max_ms": 116.793, "p95_ms": 103.784}, "visualize": {"avg_ms": 4.853, "count": 80, "max_ms": 6.796, "p95_ms": 5.528}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 540}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5460.266, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:46:51+09:00", "last_error_summary": "[h264 @ 0x55887f9342c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f9342c0] error while decoding MB 0 13, bytestream 4364 [h264 @ 0x55887f9342c0] cabac decode of qscale diff failed at 29 32 [h264 @ 0x55887f9342c0] error while decoding MB 29 32, bytestream 6152 [h264 @ 0x55887f7965c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f7965c0] error while decoding MB 0 3, bytestream 3754 [h264 @ 0x55887ecb9580] left block unavailable for requested intra", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 104, "h264_decode_error_count_per_hour": 64, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:46:51+09:00", "last_error_summary": "[h264 @ 0x55887f9342c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f9342c0] error while decoding MB 0 13, bytestream 4364 [h264 @ 0x55887f9342c0] cabac decode of qscale diff failed at 29 32 [h264 @ 0x55887f9342c0] error while decoding MB 29 32, bytestream 6152 [h264 @ 0x55887f7965c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f7965c0] error while decoding MB 0 3, bytestream 3754 [h264 @ 0x55887ecb9580] left block unavailable for requested intra", "last_exit_at": "2026-07-06T14:46:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 572.89, "last_started_at": "2026-07-06T14:37:18+09:00", "pid": 11600, "process": {"exists": false, "pid": 11600}, "restart_count": 15, "restart_count_per_hour": 7, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 132.006, "sample_count": 2648, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64915, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:46:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T14:46:51+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T14:46:51+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T14:46:51.348780+09:00", "mtime_epoch": 1783316811.3487802, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:46:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1467.84, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.712, "pushed": 2.712, "read": 2.678}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.397, "count": 81, "max_ms": 8.205, "p95_ms": 7.308}, "gstreamer_push": {"avg_ms": 1.256, "count": 81, "max_ms": 2.866, "p95_ms": 1.495}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.694, "count": 81, "max_ms": 81.181, "p95_ms": 77.087}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.851, "count": 81, "max_ms": 116.782, "p95_ms": 102.275}, "yolo11_640_fire_11class": {"avg_ms": 61.334, "count": 81, "max_ms": 85.548, "p95_ms": 80.305}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.63, "count": 81, "max_ms": 103.772, "p95_ms": 97.898}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.263, "count": 81, "max_ms": 169.659, "p95_ms": 159.697}, "read_interval": {"avg_ms": 373.939, "count": 80, "max_ms": 19591.331, "p95_ms": 169.975}, "triton_wait": {"avg_ms": 76.5, "count": 81, "max_ms": 116.793, "p95_ms": 103.784}, "visualize": {"avg_ms": 4.837, "count": 81, "max_ms": 6.796, "p95_ms": 5.528}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 540}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5452.531, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 104, "h264_decode_error_count_per_hour": 64, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:46:51+09:00", "last_error_summary": "[h264 @ 0x55887f9342c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f9342c0] error while decoding MB 0 13, bytestream 4364 [h264 @ 0x55887f9342c0] cabac decode of qscale diff failed at 29 32 [h264 @ 0x55887f9342c0] error while decoding MB 29 32, bytestream 6152 [h264 @ 0x55887f7965c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f7965c0] error while decoding MB 0 3, bytestream 3754 [h264 @ 0x55887ecb9580] left block unavailable for requested intra", "last_exit_at": "2026-07-06T14:46:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 572.897, "last_started_at": "2026-07-06T14:37:18+09:00", "pid": 11600, "process": {"exists": false, "pid": 11600}, "restart_count": 15, "restart_count_per_hour": 7, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 132.006, "sample_count": 2648, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64915, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:46:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T14:46:51+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T14:46:51+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-06T14:46:51.348780+09:00", "mtime_epoch": 1783316811.3487802, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:46:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1467.84, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.711, "pushed": 2.711, "read": 2.677}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.397, "count": 81, "max_ms": 8.205, "p95_ms": 7.308}, "gstreamer_push": {"avg_ms": 1.256, "count": 81, "max_ms": 2.866, "p95_ms": 1.495}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.694, "count": 81, "max_ms": 81.181, "p95_ms": 77.087}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.851, "count": 81, "max_ms": 116.782, "p95_ms": 102.275}, "yolo11_640_fire_11class": {"avg_ms": 61.334, "count": 81, "max_ms": 85.548, "p95_ms": 80.305}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.63, "count": 81, "max_ms": 103.772, "p95_ms": 97.898}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.263, "count": 81, "max_ms": 169.659, "p95_ms": 159.697}, "read_interval": {"avg_ms": 373.939, "count": 80, "max_ms": 19591.331, "p95_ms": 169.975}, "triton_wait": {"avg_ms": 76.5, "count": 81, "max_ms": 116.793, "p95_ms": 103.784}, "visualize": {"avg_ms": 4.837, "count": 81, "max_ms": 6.796, "p95_ms": 5.528}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 540}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5568.191, "limit_mb": 8796093022207.996}, "detail": {"pid": 11707, "restart_count": 16, "start_count": 17}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 104, "h264_decode_error_count_per_hour": 64, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:46:51+09:00", "last_error_summary": "[h264 @ 0x55887f9342c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f9342c0] error while decoding MB 0 13, bytestream 4364 [h264 @ 0x55887f9342c0] cabac decode of qscale diff failed at 29 32 [h264 @ 0x55887f9342c0] error while decoding MB 29 32, bytestream 6152 [h264 @ 0x55887f7965c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55887f7965c0] error while decoding MB 0 3, bytestream 3754 [h264 @ 0x55887ecb9580] left block unavailable for requested intra", "last_exit_at": "2026-07-06T14:46:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:46:53+09:00", "pid": 11707, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.699, "name": "ffmpeg", "pid": 11707, "rss_mb": 136.699, "state": "S (sleeping)", "threads": 106}, "restart_count": 16, "restart_count_per_hour": 8, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 132.006, "sample_count": 2613, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64915, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:46:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.093, "last_frame_pushed_at": "2026-07-06T14:46:51+09:00", "last_frame_read_age_sec": 4.203, "last_frame_read_at": "2026-07-06T14:46:51+09:00", "mjpeg": {"age_sec": 4.077, "exists": true, "mtime": "2026-07-06T14:46:51.468782+09:00", "mtime_epoch": 1783316811.4687822, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196652}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:46:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1467.84, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.741, "pushed": 1.741, "read": 1.707}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.057, "count": 52, "max_ms": 8.205, "p95_ms": 7.35}, "gstreamer_push": {"avg_ms": 1.291, "count": 52, "max_ms": 2.866, "p95_ms": 1.495}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.348, "count": 52, "max_ms": 81.181, "p95_ms": 78.322}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.712, "count": 52, "max_ms": 116.782, "p95_ms": 108.364}, "yolo11_640_fire_11class": {"avg_ms": 62.217, "count": 52, "max_ms": 84.77, "p95_ms": 80.593}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.553, "count": 52, "max_ms": 101.488, "p95_ms": 98.467}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.482, "count": 52, "max_ms": 169.659, "p95_ms": 163.311}, "read_interval": {"avg_ms": 506.166, "count": 51, "max_ms": 19591.331, "p95_ms": 174.627}, "triton_wait": {"avg_ms": 76.311, "count": 52, "max_ms": 116.793, "p95_ms": 108.377}, "visualize": {"avg_ms": 4.625, "count": 52, "max_ms": 6.796, "p95_ms": 5.528}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 471}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5397.598, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x565283548400] error while decoding MB 34 66, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:47:52+09:00", "last_error_summary": "[h264 @ 0x565283548400] error while decoding MB 34 66, bytestream -7", "last_exit_at": "2026-07-06T14:32:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 926.326, "last_started_at": "2026-07-06T14:32:25+09:00", "pid": 11448, "process": {"exists": false, "pid": 11448}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.005, "sample_count": 2458, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:47:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.001, "last_frame_pushed_at": "2026-07-06T14:47:52+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-06T14:47:52+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-06T14:47:52.249772+09:00", "mtime_epoch": 1783316872.2497718, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217281}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:47:52+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1323.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.479, "pushed": 8.479, "read": 8.446}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.131, "count": 254, "max_ms": 8.935, "p95_ms": 6.274}, "gstreamer_push": {"avg_ms": 1.465, "count": 254, "max_ms": 6.309, "p95_ms": 3.502}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.249, "count": 254, "max_ms": 89.889, "p95_ms": 80.89}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.955, "count": 254, "max_ms": 128.113, "p95_ms": 108.303}, "yolo11_640_fire_11class": {"avg_ms": 72.511, "count": 254, "max_ms": 92.241, "p95_ms": 84.688}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.254, "count": 254, "max_ms": 105.883, "p95_ms": 99.471}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.023, "count": 254, "max_ms": 154.726, "p95_ms": 133.436}, "read_interval": {"avg_ms": 118.451, "count": 253, "max_ms": 164.385, "p95_ms": 144.568}, "triton_wait": {"avg_ms": 89.301, "count": 254, "max_ms": 128.125, "p95_ms": 108.315}, "visualize": {"avg_ms": 3.823, "count": 254, "max_ms": 6.069, "p95_ms": 5.12}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 471}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5397.383, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:47:52+09:00", "last_error_summary": "[h264 @ 0x565283548400] error while decoding MB 34 66, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:47:52+09:00", "last_error_summary": "[h264 @ 0x565283548400] error while decoding MB 34 66, bytestream -7", "last_exit_at": "2026-07-06T14:47:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 926.346, "last_started_at": "2026-07-06T14:32:25+09:00", "pid": 11448, "process": {"exists": false, "pid": 11448}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.005, "sample_count": 2457, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:47:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-06T14:47:52+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-06T14:47:52+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-06T14:47:52.249772+09:00", "mtime_epoch": 1783316872.2497718, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217281}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:47:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1323.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.475, "pushed": 8.475, "read": 8.442}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.131, "count": 254, "max_ms": 8.935, "p95_ms": 6.274}, "gstreamer_push": {"avg_ms": 1.465, "count": 254, "max_ms": 6.309, "p95_ms": 3.502}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.249, "count": 254, "max_ms": 89.889, "p95_ms": 80.89}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.955, "count": 254, "max_ms": 128.113, "p95_ms": 108.303}, "yolo11_640_fire_11class": {"avg_ms": 72.511, "count": 254, "max_ms": 92.241, "p95_ms": 84.688}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.254, "count": 254, "max_ms": 105.883, "p95_ms": 99.471}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.023, "count": 254, "max_ms": 154.726, "p95_ms": 133.436}, "read_interval": {"avg_ms": 118.451, "count": 253, "max_ms": 164.385, "p95_ms": 144.568}, "triton_wait": {"avg_ms": 89.301, "count": 254, "max_ms": 128.125, "p95_ms": 108.315}, "visualize": {"avg_ms": 3.823, "count": 254, "max_ms": 6.069, "p95_ms": 5.12}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 471}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5397.375, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:47:52+09:00", "last_error_summary": "[h264 @ 0x565283548400] error while decoding MB 34 66, bytestream -7", "last_exit_at": "2026-07-06T14:47:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 926.355, "last_started_at": "2026-07-06T14:32:25+09:00", "pid": 11448, "process": {"exists": false, "pid": 11448}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.005, "sample_count": 2457, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:47:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.03, "last_frame_pushed_at": "2026-07-06T14:47:52+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-06T14:47:52+09:00", "mjpeg": {"age_sec": 0.01, "exists": true, "mtime": "2026-07-06T14:47:52.337773+09:00", "mtime_epoch": 1783316872.3377733, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231318}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:47:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1323.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.473, "pushed": 8.473, "read": 8.439}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.131, "count": 254, "max_ms": 8.935, "p95_ms": 6.274}, "gstreamer_push": {"avg_ms": 1.465, "count": 254, "max_ms": 6.309, "p95_ms": 3.502}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.249, "count": 254, "max_ms": 89.889, "p95_ms": 80.89}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.955, "count": 254, "max_ms": 128.113, "p95_ms": 108.303}, "yolo11_640_fire_11class": {"avg_ms": 72.511, "count": 254, "max_ms": 92.241, "p95_ms": 84.688}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.254, "count": 254, "max_ms": 105.883, "p95_ms": 99.471}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.023, "count": 254, "max_ms": 154.726, "p95_ms": 133.436}, "read_interval": {"avg_ms": 118.451, "count": 253, "max_ms": 164.385, "p95_ms": 144.568}, "triton_wait": {"avg_ms": 89.301, "count": 254, "max_ms": 128.125, "p95_ms": 108.315}, "visualize": {"avg_ms": 3.823, "count": 254, "max_ms": 6.069, "p95_ms": 5.12}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 471}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5509.203, "limit_mb": 8796093022207.996}, "detail": {"pid": 11814, "restart_count": 22, "start_count": 23}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:47:52+09:00", "last_error_summary": "[h264 @ 0x565283548400] error while decoding MB 34 66, bytestream -7", "last_exit_at": "2026-07-06T14:47:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:47:54+09:00", "pid": 11814, "process": {"exists": true, "fd_count": 5, "hwm_mb": 145.684, "name": "ffmpeg", "pid": 11814, "rss_mb": 145.684, "state": "S (sleeping)", "threads": 37}, "restart_count": 22, "restart_count_per_hour": 9, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.831, "sample_count": 2426, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:47:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.426, "last_frame_pushed_at": "2026-07-06T14:47:52+09:00", "last_frame_read_age_sec": 4.503, "last_frame_read_at": "2026-07-06T14:47:52+09:00", "mjpeg": {"age_sec": 4.406, "exists": true, "mtime": "2026-07-06T14:47:52.337773+09:00", "mtime_epoch": 1783316872.3377733, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231318}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:47:56+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1323.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.201, "pushed": 7.201, "read": 7.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.155, "count": 216, "max_ms": 8.935, "p95_ms": 6.301}, "gstreamer_push": {"avg_ms": 1.466, "count": 216, "max_ms": 6.309, "p95_ms": 3.56}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.461, "count": 216, "max_ms": 89.889, "p95_ms": 80.919}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.751, "count": 216, "max_ms": 128.113, "p95_ms": 108.759}, "yolo11_640_fire_11class": {"avg_ms": 72.16, "count": 216, "max_ms": 92.241, "p95_ms": 85.597}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.247, "count": 216, "max_ms": 105.883, "p95_ms": 99.481}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.138, "count": 216, "max_ms": 154.726, "p95_ms": 134.746}, "read_interval": {"avg_ms": 118.568, "count": 216, "max_ms": 164.385, "p95_ms": 146.786}, "triton_wait": {"avg_ms": 88.937, "count": 216, "max_ms": 128.125, "p95_ms": 108.771}, "visualize": {"avg_ms": 3.842, "count": 216, "max_ms": 6.069, "p95_ms": 5.132}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 477}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5419.551, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c642ff1ac0] error while decoding MB 52 45, bytestream -27 [in#0/rtsp @ 0x55c642e33e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c642e33e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:49:47+09:00", "last_error_summary": "[h264 @ 0x55c642ff1ac0] error while decoding MB 52 45, bytestream -27 [in#0/rtsp @ 0x55c642e33e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c642e33e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:47:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.784, "last_started_at": "2026-07-06T14:47:54+09:00", "pid": 11814, "process": {"exists": false, "pid": 11814}, "restart_count": 22, "restart_count_per_hour": 9, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.221, "sample_count": 2273, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57242, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:49:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T14:49:47+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T14:49:47+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T14:49:47.475679+09:00", "mtime_epoch": 1783316987.4756787, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228889}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:49:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1327.348, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 32, "pushed_frames": 32, "read_frames": 33, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.57, "pushed": 1.57, "read": 1.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.531, "count": 32, "max_ms": 7.31, "p95_ms": 6.78}, "gstreamer_push": {"avg_ms": 1.167, "count": 32, "max_ms": 1.449, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.859, "count": 32, "max_ms": 83.285, "p95_ms": 79.973}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.666, "count": 32, "max_ms": 122.421, "p95_ms": 110.075}, "yolo11_640_fire_11class": {"avg_ms": 63.388, "count": 32, "max_ms": 89.31, "p95_ms": 85.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.086, "count": 32, "max_ms": 93.543, "p95_ms": 93.292}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.074, "count": 32, "max_ms": 172.742, "p95_ms": 168.691}, "read_interval": {"avg_ms": 1096.058, "count": 33, "max_ms": 16391.739, "p95_ms": 15859.583}, "triton_wait": {"avg_ms": 76.214, "count": 32, "max_ms": 122.433, "p95_ms": 110.087}, "visualize": {"avg_ms": 4.669, "count": 32, "max_ms": 5.375, "p95_ms": 5.294}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 20.376, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 477}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5418.836, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:49:47+09:00", "last_error_summary": "[h264 @ 0x55c642ff1ac0] error while decoding MB 52 45, bytestream -27 [in#0/rtsp @ 0x55c642e33e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c642e33e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:49:47+09:00", "last_error_summary": "[h264 @ 0x55c642ff1ac0] error while decoding MB 52 45, bytestream -27 [in#0/rtsp @ 0x55c642e33e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c642e33e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:49:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.852, "last_started_at": "2026-07-06T14:47:54+09:00", "pid": 11814, "process": {"exists": false, "pid": 11814}, "restart_count": 22, "restart_count_per_hour": 9, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.218, "sample_count": 2273, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:49:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T14:49:47+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T14:49:47+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T14:49:47.475679+09:00", "mtime_epoch": 1783316987.4756787, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228889}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:49:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1327.348, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 33, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.614, "pushed": 1.614, "read": 1.614}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.662, "count": 33, "max_ms": 9.876, "p95_ms": 7.31}, "gstreamer_push": {"avg_ms": 1.168, "count": 33, "max_ms": 1.449, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.858, "count": 33, "max_ms": 83.285, "p95_ms": 79.973}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.419, "count": 33, "max_ms": 122.421, "p95_ms": 110.075}, "yolo11_640_fire_11class": {"avg_ms": 63.148, "count": 33, "max_ms": 89.31, "p95_ms": 85.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.69, "count": 33, "max_ms": 93.543, "p95_ms": 93.292}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.351, "count": 33, "max_ms": 172.742, "p95_ms": 168.691}, "read_interval": {"avg_ms": 1096.058, "count": 33, "max_ms": 16391.739, "p95_ms": 15859.583}, "triton_wait": {"avg_ms": 75.951, "count": 33, "max_ms": 122.433, "p95_ms": 110.087}, "visualize": {"avg_ms": 4.714, "count": 33, "max_ms": 6.153, "p95_ms": 5.375}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 20.444, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 477}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5418.82, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:49:47+09:00", "last_error_summary": "[h264 @ 0x55c642ff1ac0] error while decoding MB 52 45, bytestream -27 [in#0/rtsp @ 0x55c642e33e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c642e33e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:49:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.858, "last_started_at": "2026-07-06T14:47:54+09:00", "pid": 11814, "process": {"exists": false, "pid": 11814}, "restart_count": 22, "restart_count_per_hour": 9, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.218, "sample_count": 2273, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:49:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T14:49:47+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T14:49:47+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-06T14:49:47.475679+09:00", "mtime_epoch": 1783316987.4756787, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228889}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:49:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1327.348, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 33, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.614, "pushed": 1.614, "read": 1.614}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.662, "count": 33, "max_ms": 9.876, "p95_ms": 7.31}, "gstreamer_push": {"avg_ms": 1.168, "count": 33, "max_ms": 1.449, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.858, "count": 33, "max_ms": 83.285, "p95_ms": 79.973}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.419, "count": 33, "max_ms": 122.421, "p95_ms": 110.075}, "yolo11_640_fire_11class": {"avg_ms": 63.148, "count": 33, "max_ms": 89.31, "p95_ms": 85.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.69, "count": 33, "max_ms": 93.543, "p95_ms": 93.292}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.351, "count": 33, "max_ms": 172.742, "p95_ms": 168.691}, "read_interval": {"avg_ms": 1096.058, "count": 33, "max_ms": 16391.739, "p95_ms": 15859.583}, "triton_wait": {"avg_ms": 75.951, "count": 33, "max_ms": 122.433, "p95_ms": 110.087}, "visualize": {"avg_ms": 4.714, "count": 33, "max_ms": 6.153, "p95_ms": 5.375}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 20.45, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 477}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5585.559, "limit_mb": 8796093022207.996}, "detail": {"pid": 11966, "restart_count": 23, "start_count": 24}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:49:47+09:00", "last_error_summary": "[h264 @ 0x55c642ff1ac0] error while decoding MB 52 45, bytestream -27 [in#0/rtsp @ 0x55c642e33e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c642e33e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:49:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:49:50+09:00", "pid": 11966, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.516, "name": "ffmpeg", "pid": 11966, "rss_mb": 204.516, "state": "S (sleeping)", "threads": 106}, "restart_count": 23, "restart_count_per_hour": 10, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.926, "sample_count": 2241, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:49:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.439, "last_frame_pushed_at": "2026-07-06T14:49:47+09:00", "last_frame_read_age_sec": 4.568, "last_frame_read_at": "2026-07-06T14:49:47+09:00", "mjpeg": {"age_sec": 4.421, "exists": true, "mtime": "2026-07-06T14:49:47.607681+09:00", "mtime_epoch": 1783316987.6076808, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228889}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:49:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1582.109, "name": "celery", "pid": 267, "rss_mb": 1327.348, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 33, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.326, "pushed": 1.326, "read": 1.326}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.662, "count": 33, "max_ms": 9.876, "p95_ms": 7.31}, "gstreamer_push": {"avg_ms": 1.168, "count": 33, "max_ms": 1.449, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.858, "count": 33, "max_ms": 83.285, "p95_ms": 79.973}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.419, "count": 33, "max_ms": 122.421, "p95_ms": 110.075}, "yolo11_640_fire_11class": {"avg_ms": 63.148, "count": 33, "max_ms": 89.31, "p95_ms": 85.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.69, "count": 33, "max_ms": 93.543, "p95_ms": 93.292}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.351, "count": 33, "max_ms": 172.742, "p95_ms": 168.691}, "read_interval": {"avg_ms": 1096.058, "count": 33, "max_ms": 16391.739, "p95_ms": 15859.583}, "triton_wait": {"avg_ms": 75.951, "count": 33, "max_ms": 122.433, "p95_ms": 110.087}, "visualize": {"avg_ms": 4.714, "count": 33, "max_ms": 6.153, "p95_ms": 5.375}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 24.879, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 491}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5450.777, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56334507bd00] error while decoding MB 57 32, bytestream -25 [in#0/rtsp @ 0x56334429fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56334429fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:53:36+09:00", "last_error_summary": "[h264 @ 0x56334507bd00] error while decoding MB 57 32, bytestream -25 [in#0/rtsp @ 0x56334429fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56334429fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:49:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.943, "last_started_at": "2026-07-06T14:49:50+09:00", "pid": 11966, "process": {"exists": false, "pid": 11966}, "restart_count": 23, "restart_count_per_hour": 10, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.133, "sample_count": 1994, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58921, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:53:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T14:53:36+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T14:53:36+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-06T14:53:36.923568+09:00", "mtime_epoch": 1783317216.9235678, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236911}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:53:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1334.281, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.664, "pushed": 0.664, "read": 0.72}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.466, "count": 12, "max_ms": 6.654, "p95_ms": 6.654}, "gstreamer_push": {"avg_ms": 1.329, "count": 12, "max_ms": 2.689, "p95_ms": 2.689}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.033, "count": 12, "max_ms": 59.098, "p95_ms": 59.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.947, "count": 12, "max_ms": 95.14, "p95_ms": 95.14}, "yolo11_640_fire_11class": {"avg_ms": 61.525, "count": 12, "max_ms": 88.338, "p95_ms": 88.338}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.482, "count": 12, "max_ms": 76.099, "p95_ms": 76.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.373, "count": 12, "max_ms": 160.059, "p95_ms": 160.059}, "read_interval": {"avg_ms": 2446.392, "count": 13, "max_ms": 16705.557, "p95_ms": 16705.557}, "triton_wait": {"avg_ms": 64.993, "count": 12, "max_ms": 95.152, "p95_ms": 95.152}, "visualize": {"avg_ms": 4.649, "count": 12, "max_ms": 5.169, "p95_ms": 5.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.064, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 491}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5449.984, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:53:36+09:00", "last_error_summary": "[h264 @ 0x56334507bd00] error while decoding MB 57 32, bytestream -25 [in#0/rtsp @ 0x56334429fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56334429fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:53:36+09:00", "last_error_summary": "[h264 @ 0x56334507bd00] error while decoding MB 57 32, bytestream -25 [in#0/rtsp @ 0x56334429fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56334429fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:53:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 227.007, "last_started_at": "2026-07-06T14:49:50+09:00", "pid": 11966, "process": {"exists": false, "pid": 11966}, "restart_count": 23, "restart_count_per_hour": 10, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.133, "sample_count": 1994, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58922, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:53:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T14:53:37+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T14:53:36+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T14:53:36.923568+09:00", "mtime_epoch": 1783317216.9235678, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236911}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:53:37+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1334.281, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.717, "pushed": 0.717, "read": 0.717}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.777, "count": 13, "max_ms": 9.514, "p95_ms": 9.514}, "gstreamer_push": {"avg_ms": 1.319, "count": 13, "max_ms": 2.689, "p95_ms": 2.689}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.921, "count": 13, "max_ms": 59.098, "p95_ms": 59.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.498, "count": 13, "max_ms": 95.14, "p95_ms": 95.14}, "yolo11_640_fire_11class": {"avg_ms": 61.368, "count": 13, "max_ms": 88.338, "p95_ms": 88.338}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.46, "count": 13, "max_ms": 76.099, "p95_ms": 76.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.415, "count": 13, "max_ms": 160.059, "p95_ms": 160.059}, "read_interval": {"avg_ms": 2446.392, "count": 13, "max_ms": 16705.557, "p95_ms": 16705.557}, "triton_wait": {"avg_ms": 64.626, "count": 13, "max_ms": 95.152, "p95_ms": 95.152}, "visualize": {"avg_ms": 4.776, "count": 13, "max_ms": 6.303, "p95_ms": 6.303}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.126, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 491}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5449.977, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:53:36+09:00", "last_error_summary": "[h264 @ 0x56334507bd00] error while decoding MB 57 32, bytestream -25 [in#0/rtsp @ 0x56334429fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56334429fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:53:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 227.012, "last_started_at": "2026-07-06T14:49:50+09:00", "pid": 11966, "process": {"exists": false, "pid": 11966}, "restart_count": 23, "restart_count_per_hour": 10, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.133, "sample_count": 1994, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58922, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:53:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T14:53:37+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-06T14:53:36+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T14:53:36.923568+09:00", "mtime_epoch": 1783317216.9235678, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236911}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:53:37+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1334.281, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.717, "pushed": 0.717, "read": 0.717}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.777, "count": 13, "max_ms": 9.514, "p95_ms": 9.514}, "gstreamer_push": {"avg_ms": 1.319, "count": 13, "max_ms": 2.689, "p95_ms": 2.689}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.921, "count": 13, "max_ms": 59.098, "p95_ms": 59.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.498, "count": 13, "max_ms": 95.14, "p95_ms": 95.14}, "yolo11_640_fire_11class": {"avg_ms": 61.368, "count": 13, "max_ms": 88.338, "p95_ms": 88.338}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.46, "count": 13, "max_ms": 76.099, "p95_ms": 76.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.415, "count": 13, "max_ms": 160.059, "p95_ms": 160.059}, "read_interval": {"avg_ms": 2446.392, "count": 13, "max_ms": 16705.557, "p95_ms": 16705.557}, "triton_wait": {"avg_ms": 64.626, "count": 13, "max_ms": 95.152, "p95_ms": 95.152}, "visualize": {"avg_ms": 4.776, "count": 13, "max_ms": 6.303, "p95_ms": 6.303}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.131, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 491}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5632.855, "limit_mb": 8796093022207.996}, "detail": {"pid": 12178, "restart_count": 24, "start_count": 25}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:53:36+09:00", "last_error_summary": "[h264 @ 0x56334507bd00] error while decoding MB 57 32, bytestream -25 [in#0/rtsp @ 0x56334429fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56334429fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:53:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:53:39+09:00", "pid": 12178, "process": {"exists": true, "fd_count": 5, "hwm_mb": 207.125, "name": "ffmpeg", "pid": 12178, "rss_mb": 207.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 24, "restart_count_per_hour": 11, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.149, "sample_count": 1958, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58922, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:53:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.21, "last_frame_pushed_at": "2026-07-06T14:53:37+09:00", "last_frame_read_age_sec": 4.331, "last_frame_read_at": "2026-07-06T14:53:36+09:00", "mjpeg": {"age_sec": 4.189, "exists": true, "mtime": "2026-07-06T14:53:37.051570+09:00", "mtime_epoch": 1783317217.05157, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236511}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:53:41+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1334.281, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.582, "pushed": 0.582, "read": 0.582}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.777, "count": 13, "max_ms": 9.514, "p95_ms": 9.514}, "gstreamer_push": {"avg_ms": 1.319, "count": 13, "max_ms": 2.689, "p95_ms": 2.689}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.921, "count": 13, "max_ms": 59.098, "p95_ms": 59.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.498, "count": 13, "max_ms": 95.14, "p95_ms": 95.14}, "yolo11_640_fire_11class": {"avg_ms": 61.368, "count": 13, "max_ms": 88.338, "p95_ms": 88.338}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.46, "count": 13, "max_ms": 76.099, "p95_ms": 76.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.415, "count": 13, "max_ms": 160.059, "p95_ms": 160.059}, "read_interval": {"avg_ms": 2446.392, "count": 13, "max_ms": 16705.557, "p95_ms": 16705.557}, "triton_wait": {"avg_ms": 64.626, "count": 13, "max_ms": 95.152, "p95_ms": 95.152}, "visualize": {"avg_ms": 4.776, "count": 13, "max_ms": 6.303, "p95_ms": 6.303}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.332, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 571}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5484.445, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 11, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555a58520a80] cabac decode of qscale diff failed at 11 36 [h264 @ 0x555a58520a80] error while decoding MB 11 36, bytestream 4521 [h264 @ 0x555a57927140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555a57927140] error while decoding MB 0 36, bytestream 1193 [h264 @ 0x555a5797a700] cabac decode of qscale diff failed at 62 31 [h264 @ 0x555a5797a700] error while decoding MB 62 31, bytestream 9426 [h264 @ 0x555a58451c00] left block unavailable for requested intra mode [h", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 115, "h264_decode_error_count_per_hour": 75, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:54:07+09:00", "last_error_summary": "[h264 @ 0x555a58520a80] cabac decode of qscale diff failed at 11 36 [h264 @ 0x555a58520a80] error while decoding MB 11 36, bytestream 4521 [h264 @ 0x555a57927140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555a57927140] error while decoding MB 0 36, bytestream 1193 [h264 @ 0x555a5797a700] cabac decode of qscale diff failed at 62 31 [h264 @ 0x555a5797a700] error while decoding MB 62 31, bytestream 9426 [h264 @ 0x555a58451c00] left block unavailable for requested intra mode [h", "last_exit_at": "2026-07-06T14:46:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 434.212, "last_started_at": "2026-07-06T14:46:53+09:00", "pid": 11707, "process": {"exists": false, "pid": 11707}, "restart_count": 16, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.176, "sample_count": 2414, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68525, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:54:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T14:54:07+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T14:54:07+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T14:54:07.700097+09:00", "mtime_epoch": 1783317247.7000966, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:54:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1474.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 108, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.6, "pushed": 3.6, "read": 3.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.507, "count": 107, "max_ms": 8.893, "p95_ms": 7.185}, "gstreamer_push": {"avg_ms": 1.193, "count": 108, "max_ms": 3.677, "p95_ms": 1.367}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.186, "count": 107, "max_ms": 85.505, "p95_ms": 75.599}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.955, "count": 107, "max_ms": 117.529, "p95_ms": 103.233}, "yolo11_640_fire_11class": {"avg_ms": 59.754, "count": 107, "max_ms": 87.37, "p95_ms": 78.733}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.496, "count": 107, "max_ms": 103.615, "p95_ms": 93.293}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.507, "count": 108, "max_ms": 174.295, "p95_ms": 157.175}, "read_interval": {"avg_ms": 278.281, "count": 108, "max_ms": 10209.028, "p95_ms": 171.568}, "triton_wait": {"avg_ms": 73.694, "count": 107, "max_ms": 117.542, "p95_ms": 103.246}, "visualize": {"avg_ms": 5.205, "count": 107, "max_ms": 8.655, "p95_ms": 6.053}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 571}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5477.434, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:54:07+09:00", "last_error_summary": "[h264 @ 0x555a58520a80] cabac decode of qscale diff failed at 11 36 [h264 @ 0x555a58520a80] error while decoding MB 11 36, bytestream 4521 [h264 @ 0x555a57927140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555a57927140] error while decoding MB 0 36, bytestream 1193 [h264 @ 0x555a5797a700] cabac decode of qscale diff failed at 62 31 [h264 @ 0x555a5797a700] error while decoding MB 62 31, bytestream 9426 [h264 @ 0x555a58451c00] left block unavailable for requested intra mode [h", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 115, "h264_decode_error_count_per_hour": 75, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:54:07+09:00", "last_error_summary": "[h264 @ 0x555a58520a80] cabac decode of qscale diff failed at 11 36 [h264 @ 0x555a58520a80] error while decoding MB 11 36, bytestream 4521 [h264 @ 0x555a57927140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555a57927140] error while decoding MB 0 36, bytestream 1193 [h264 @ 0x555a5797a700] cabac decode of qscale diff failed at 62 31 [h264 @ 0x555a5797a700] error while decoding MB 62 31, bytestream 9426 [h264 @ 0x555a58451c00] left block unavailable for requested intra mode [h", "last_exit_at": "2026-07-06T14:54:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 434.297, "last_started_at": "2026-07-06T14:46:53+09:00", "pid": 11707, "process": {"exists": false, "pid": 11707}, "restart_count": 16, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.176, "sample_count": 2414, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68526, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:54:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T14:54:07+09:00", "last_frame_read_age_sec": 0.15, "last_frame_read_at": "2026-07-06T14:54:07+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T14:54:07.700097+09:00", "mtime_epoch": 1783317247.7000966, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:54:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1474.262, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.606, "pushed": 3.606, "read": 3.572}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.506, "count": 108, "max_ms": 8.893, "p95_ms": 7.185}, "gstreamer_push": {"avg_ms": 1.194, "count": 108, "max_ms": 3.677, "p95_ms": 1.367}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.271, "count": 108, "max_ms": 85.505, "p95_ms": 75.599}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.142, "count": 108, "max_ms": 117.529, "p95_ms": 103.233}, "yolo11_640_fire_11class": {"avg_ms": 59.904, "count": 108, "max_ms": 87.37, "p95_ms": 78.733}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.742, "count": 108, "max_ms": 103.615, "p95_ms": 93.293}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.698, "count": 108, "max_ms": 174.295, "p95_ms": 157.175}, "read_interval": {"avg_ms": 279.63, "count": 107, "max_ms": 10209.028, "p95_ms": 171.568}, "triton_wait": {"avg_ms": 73.874, "count": 108, "max_ms": 117.542, "p95_ms": 103.246}, "visualize": {"avg_ms": 5.203, "count": 108, "max_ms": 8.655, "p95_ms": 6.053}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 571}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5477.363, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 115, "h264_decode_error_count_per_hour": 75, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:54:07+09:00", "last_error_summary": "[h264 @ 0x555a58520a80] cabac decode of qscale diff failed at 11 36 [h264 @ 0x555a58520a80] error while decoding MB 11 36, bytestream 4521 [h264 @ 0x555a57927140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555a57927140] error while decoding MB 0 36, bytestream 1193 [h264 @ 0x555a5797a700] cabac decode of qscale diff failed at 62 31 [h264 @ 0x555a5797a700] error while decoding MB 62 31, bytestream 9426 [h264 @ 0x555a58451c00] left block unavailable for requested intra mode [h", "last_exit_at": "2026-07-06T14:54:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 434.302, "last_started_at": "2026-07-06T14:46:53+09:00", "pid": 11707, "process": {"exists": false, "pid": 11707}, "restart_count": 16, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.176, "sample_count": 2414, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68526, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:54:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T14:54:07+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-06T14:54:07+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-06T14:54:07.700097+09:00", "mtime_epoch": 1783317247.7000966, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:54:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1474.262, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.605, "pushed": 3.605, "read": 3.572}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.506, "count": 108, "max_ms": 8.893, "p95_ms": 7.185}, "gstreamer_push": {"avg_ms": 1.194, "count": 108, "max_ms": 3.677, "p95_ms": 1.367}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.271, "count": 108, "max_ms": 85.505, "p95_ms": 75.599}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.142, "count": 108, "max_ms": 117.529, "p95_ms": 103.233}, "yolo11_640_fire_11class": {"avg_ms": 59.904, "count": 108, "max_ms": 87.37, "p95_ms": 78.733}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.742, "count": 108, "max_ms": 103.615, "p95_ms": 93.293}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.698, "count": 108, "max_ms": 174.295, "p95_ms": 157.175}, "read_interval": {"avg_ms": 279.63, "count": 107, "max_ms": 10209.028, "p95_ms": 171.568}, "triton_wait": {"avg_ms": 73.874, "count": 108, "max_ms": 117.542, "p95_ms": 103.246}, "visualize": {"avg_ms": 5.203, "count": 108, "max_ms": 8.655, "p95_ms": 6.053}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 571}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5589.539, "limit_mb": 8796093022207.996}, "detail": {"pid": 12330, "restart_count": 17, "start_count": 18}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 115, "h264_decode_error_count_per_hour": 74, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:54:07+09:00", "last_error_summary": "[h264 @ 0x555a58520a80] cabac decode of qscale diff failed at 11 36 [h264 @ 0x555a58520a80] error while decoding MB 11 36, bytestream 4521 [h264 @ 0x555a57927140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555a57927140] error while decoding MB 0 36, bytestream 1193 [h264 @ 0x555a5797a700] cabac decode of qscale diff failed at 62 31 [h264 @ 0x555a5797a700] error while decoding MB 62 31, bytestream 9426 [h264 @ 0x555a58451c00] left block unavailable for requested intra mode [h", "last_exit_at": "2026-07-06T14:54:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:54:10+09:00", "pid": 12330, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.184, "name": "ffmpeg", "pid": 12330, "rss_mb": 136.184, "state": "S (sleeping)", "threads": 106}, "restart_count": 17, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.432, "sample_count": 2372, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68526, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:54:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.446, "last_frame_pushed_at": "2026-07-06T14:54:07+09:00", "last_frame_read_age_sec": 4.592, "last_frame_read_at": "2026-07-06T14:54:07+09:00", "mjpeg": {"age_sec": 4.431, "exists": true, "mtime": "2026-07-06T14:54:07.852099+09:00", "mtime_epoch": 1783317247.8520994, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191208}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T14:54:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1474.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.405, "pushed": 2.405, "read": 2.372}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.669, "count": 72, "max_ms": 8.893, "p95_ms": 7.809}, "gstreamer_push": {"avg_ms": 1.148, "count": 72, "max_ms": 2.494, "p95_ms": 1.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.096, "count": 72, "max_ms": 85.505, "p95_ms": 77.602}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.233, "count": 72, "max_ms": 117.529, "p95_ms": 105.123}, "yolo11_640_fire_11class": {"avg_ms": 62.116, "count": 72, "max_ms": 87.37, "p95_ms": 83.008}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.94, "count": 72, "max_ms": 103.615, "p95_ms": 97.13}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.578, "count": 72, "max_ms": 174.295, "p95_ms": 162.395}, "read_interval": {"avg_ms": 359.218, "count": 71, "max_ms": 10209.028, "p95_ms": 182.038}, "triton_wait": {"avg_ms": 76.314, "count": 72, "max_ms": 117.542, "p95_ms": 105.136}, "visualize": {"avg_ms": 5.337, "count": 72, "max_ms": 8.655, "p95_ms": 6.369}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 501}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5439.121, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ddecdac700] error while decoding MB 58 52, bytestream -15 [in#0/rtsp @ 0x55ddec309e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ddec309e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:56:29+09:00", "last_error_summary": "[h264 @ 0x55ddecdac700] error while decoding MB 58 52, bytestream -15 [in#0/rtsp @ 0x55ddec309e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ddec309e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:53:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 170.196, "last_started_at": "2026-07-06T14:53:39+09:00", "pid": 12178, "process": {"exists": false, "pid": 12178}, "restart_count": 24, "restart_count_per_hour": 11, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.459, "sample_count": 2134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60160, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:56:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.082, "last_frame_pushed_at": "2026-07-06T14:56:29+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-06T14:56:29+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-06T14:56:29.370548+09:00", "mtime_epoch": 1783317389.3705485, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218187}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:56:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1337.371, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 36, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.202, "pushed": 1.202, "read": 1.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.591, "count": 36, "max_ms": 6.917, "p95_ms": 6.747}, "gstreamer_push": {"avg_ms": 1.18, "count": 36, "max_ms": 1.36, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.41, "count": 36, "max_ms": 66.128, "p95_ms": 65.487}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.647, "count": 36, "max_ms": 115.626, "p95_ms": 109.5}, "yolo11_640_fire_11class": {"avg_ms": 63.751, "count": 36, "max_ms": 93.097, "p95_ms": 88.825}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.721, "count": 36, "max_ms": 99.204, "p95_ms": 94.191}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.075, "count": 36, "max_ms": 166.268, "p95_ms": 152.882}, "read_interval": {"avg_ms": 833.718, "count": 36, "max_ms": 16182.129, "p95_ms": 9475.839}, "triton_wait": {"avg_ms": 76.763, "count": 36, "max_ms": 115.639, "p95_ms": 109.512}, "visualize": {"avg_ms": 4.71, "count": 36, "max_ms": 5.561, "p95_ms": 5.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 501}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5438.84, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:56:29+09:00", "last_error_summary": "[h264 @ 0x55ddecdac700] error while decoding MB 58 52, bytestream -15 [in#0/rtsp @ 0x55ddec309e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ddec309e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:56:29+09:00", "last_error_summary": "[h264 @ 0x55ddecdac700] error while decoding MB 58 52, bytestream -15 [in#0/rtsp @ 0x55ddec309e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ddec309e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:56:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 170.239, "last_started_at": "2026-07-06T14:53:39+09:00", "pid": 12178, "process": {"exists": false, "pid": 12178}, "restart_count": 24, "restart_count_per_hour": 11, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.459, "sample_count": 2134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:56:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T14:56:29+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-06T14:56:29+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T14:56:29.370548+09:00", "mtime_epoch": 1783317389.3705485, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218187}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:56:29+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1337.371, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 37, "pushed_frames": 37, "read_frames": 36, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.234, "pushed": 1.234, "read": 1.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.527, "count": 36, "max_ms": 6.917, "p95_ms": 6.747}, "gstreamer_push": {"avg_ms": 1.185, "count": 37, "max_ms": 1.373, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.01, "count": 36, "max_ms": 74.357, "p95_ms": 66.128}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.848, "count": 36, "max_ms": 115.626, "p95_ms": 109.5}, "yolo11_640_fire_11class": {"avg_ms": 64.133, "count": 36, "max_ms": 93.097, "p95_ms": 88.825}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.454, "count": 36, "max_ms": 99.204, "p95_ms": 94.191}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.663, "count": 37, "max_ms": 166.268, "p95_ms": 152.882}, "read_interval": {"avg_ms": 833.718, "count": 36, "max_ms": 16182.129, "p95_ms": 9475.839}, "triton_wait": {"avg_ms": 76.964, "count": 36, "max_ms": 115.639, "p95_ms": 109.512}, "visualize": {"avg_ms": 4.658, "count": 36, "max_ms": 5.561, "p95_ms": 5.449}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 501}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5438.957, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:56:29+09:00", "last_error_summary": "[h264 @ 0x55ddecdac700] error while decoding MB 58 52, bytestream -15 [in#0/rtsp @ 0x55ddec309e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ddec309e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:56:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 170.247, "last_started_at": "2026-07-06T14:53:39+09:00", "pid": 12178, "process": {"exists": false, "pid": 12178}, "restart_count": 24, "restart_count_per_hour": 11, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.459, "sample_count": 2134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:56:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T14:56:29+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T14:56:29+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T14:56:29.370548+09:00", "mtime_epoch": 1783317389.3705485, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218187}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:56:29+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1337.371, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 37, "pushed_frames": 37, "read_frames": 36, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.233, "pushed": 1.233, "read": 1.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.527, "count": 36, "max_ms": 6.917, "p95_ms": 6.747}, "gstreamer_push": {"avg_ms": 1.185, "count": 37, "max_ms": 1.373, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.01, "count": 36, "max_ms": 74.357, "p95_ms": 66.128}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.848, "count": 36, "max_ms": 115.626, "p95_ms": 109.5}, "yolo11_640_fire_11class": {"avg_ms": 64.133, "count": 36, "max_ms": 93.097, "p95_ms": 88.825}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.454, "count": 36, "max_ms": 99.204, "p95_ms": 94.191}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.663, "count": 37, "max_ms": 166.268, "p95_ms": 152.882}, "read_interval": {"avg_ms": 833.718, "count": 36, "max_ms": 16182.129, "p95_ms": 9475.839}, "triton_wait": {"avg_ms": 76.964, "count": 36, "max_ms": 115.639, "p95_ms": 109.512}, "visualize": {"avg_ms": 4.658, "count": 36, "max_ms": 5.561, "p95_ms": 5.449}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 501}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5616.355, "limit_mb": 8796093022207.996}, "detail": {"pid": 12437, "restart_count": 25, "start_count": 26}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:56:29+09:00", "last_error_summary": "[h264 @ 0x55ddecdac700] error while decoding MB 58 52, bytestream -15 [in#0/rtsp @ 0x55ddec309e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ddec309e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:56:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:56:32+09:00", "pid": 12437, "process": {"exists": true, "fd_count": 5, "hwm_mb": 207.078, "name": "ffmpeg", "pid": 12437, "rss_mb": 207.078, "state": "S (sleeping)", "threads": 106}, "restart_count": 25, "restart_count_per_hour": 12, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.547, "sample_count": 2090, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:56:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.68, "last_frame_pushed_at": "2026-07-06T14:56:29+09:00", "last_frame_read_age_sec": 4.784, "last_frame_read_at": "2026-07-06T14:56:29+09:00", "mjpeg": {"age_sec": 4.662, "exists": true, "mtime": "2026-07-06T14:56:29.490551+09:00", "mtime_epoch": 1783317389.4905508, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217169}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:56:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1337.371, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.584, "pushed": 0.584, "read": 0.584}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.856, "count": 13, "max_ms": 6.592, "p95_ms": 6.592}, "gstreamer_push": {"avg_ms": 1.256, "count": 13, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.331, "count": 13, "max_ms": 74.357, "p95_ms": 74.357}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.112, "count": 13, "max_ms": 115.626, "p95_ms": 115.626}, "yolo11_640_fire_11class": {"avg_ms": 65.487, "count": 13, "max_ms": 93.097, "p95_ms": 93.097}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.73, "count": 13, "max_ms": 99.204, "p95_ms": 99.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.862, "count": 13, "max_ms": 166.268, "p95_ms": 166.268}, "read_interval": {"avg_ms": 2071.969, "count": 13, "max_ms": 16182.129, "p95_ms": 16182.129}, "triton_wait": {"avg_ms": 79.194, "count": 13, "max_ms": 115.639, "p95_ms": 115.639}, "visualize": {"avg_ms": 4.233, "count": 13, "max_ms": 5.561, "p95_ms": 5.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.245, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5434.309, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e82838d680] error while decoding MB 118 20, bytestream -35 [in#0/rtsp @ 0x55e827a8be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e827a8be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:58:52+09:00", "last_error_summary": "[h264 @ 0x55e82838d680] error while decoding MB 118 20, bytestream -35 [in#0/rtsp @ 0x55e827a8be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e827a8be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:56:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.846, "last_started_at": "2026-07-06T14:56:32+09:00", "pid": 12437, "process": {"exists": false, "pid": 12437}, "restart_count": 25, "restart_count_per_hour": 12, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.421, "sample_count": 2048, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:58:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T14:58:52+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-06T14:58:52+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-06T14:58:52.945057+09:00", "mtime_epoch": 1783317532.9450567, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227111}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:58:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1331.855, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.869, "pushed": 0.869, "read": 0.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.047, "count": 26, "max_ms": 9.526, "p95_ms": 9.243}, "gstreamer_push": {"avg_ms": 1.442, "count": 26, "max_ms": 6.174, "p95_ms": 1.476}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.485, "count": 26, "max_ms": 79.708, "p95_ms": 77.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.019, "count": 26, "max_ms": 111.858, "p95_ms": 106.27}, "yolo11_640_fire_11class": {"avg_ms": 67.26, "count": 26, "max_ms": 86.43, "p95_ms": 81.347}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.646, "count": 26, "max_ms": 101.395, "p95_ms": 93.741}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.238, "count": 26, "max_ms": 146.47, "p95_ms": 145.968}, "read_interval": {"avg_ms": 1152.14, "count": 26, "max_ms": 16023.613, "p95_ms": 11134.256}, "triton_wait": {"avg_ms": 79.444, "count": 26, "max_ms": 111.871, "p95_ms": 106.283}, "visualize": {"avg_ms": 4.159, "count": 26, "max_ms": 5.589, "p95_ms": 5.463}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5436.188, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T14:58:52+09:00", "last_error_summary": "[h264 @ 0x55e82838d680] error while decoding MB 118 20, bytestream -35 [in#0/rtsp @ 0x55e827a8be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e827a8be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:58:52+09:00", "last_error_summary": "[h264 @ 0x55e82838d680] error while decoding MB 118 20, bytestream -35 [in#0/rtsp @ 0x55e827a8be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e827a8be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:58:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.881, "last_started_at": "2026-07-06T14:56:32+09:00", "pid": 12437, "process": {"exists": false, "pid": 12437}, "restart_count": 25, "restart_count_per_hour": 12, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.421, "sample_count": 2049, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:58:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T14:58:53+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-06T14:58:52+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-06T14:58:52.945057+09:00", "mtime_epoch": 1783317532.9450567, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227111}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:58:53+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1331.855, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 27, "pushed_frames": 27, "read_frames": 26, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.902, "pushed": 0.902, "read": 0.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.998, "count": 27, "max_ms": 9.526, "p95_ms": 9.243}, "gstreamer_push": {"avg_ms": 1.436, "count": 27, "max_ms": 6.174, "p95_ms": 1.476}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.963, "count": 27, "max_ms": 79.708, "p95_ms": 77.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.215, "count": 27, "max_ms": 111.858, "p95_ms": 106.27}, "yolo11_640_fire_11class": {"avg_ms": 67.522, "count": 27, "max_ms": 86.43, "p95_ms": 81.347}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.706, "count": 27, "max_ms": 101.395, "p95_ms": 93.741}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.913, "count": 27, "max_ms": 146.47, "p95_ms": 145.968}, "read_interval": {"avg_ms": 1152.14, "count": 26, "max_ms": 16023.613, "p95_ms": 11134.256}, "triton_wait": {"avg_ms": 79.624, "count": 27, "max_ms": 111.871, "p95_ms": 106.283}, "visualize": {"avg_ms": 4.137, "count": 27, "max_ms": 5.589, "p95_ms": 5.463}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5436.277, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:58:52+09:00", "last_error_summary": "[h264 @ 0x55e82838d680] error while decoding MB 118 20, bytestream -35 [in#0/rtsp @ 0x55e827a8be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e827a8be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:58:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.888, "last_started_at": "2026-07-06T14:56:32+09:00", "pid": 12437, "process": {"exists": false, "pid": 12437}, "restart_count": 25, "restart_count_per_hour": 12, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.421, "sample_count": 2048, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:58:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T14:58:53+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T14:58:52+09:00", "mjpeg": {"age_sec": 0.007, "exists": true, "mtime": "2026-07-06T14:58:53.033058+09:00", "mtime_epoch": 1783317533.0330584, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222743}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:58:53+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1331.855, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 27, "pushed_frames": 27, "read_frames": 26, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.901, "pushed": 0.901, "read": 0.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.998, "count": 27, "max_ms": 9.526, "p95_ms": 9.243}, "gstreamer_push": {"avg_ms": 1.436, "count": 27, "max_ms": 6.174, "p95_ms": 1.476}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.963, "count": 27, "max_ms": 79.708, "p95_ms": 77.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.215, "count": 27, "max_ms": 111.858, "p95_ms": 106.27}, "yolo11_640_fire_11class": {"avg_ms": 67.522, "count": 27, "max_ms": 86.43, "p95_ms": 81.347}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.706, "count": 27, "max_ms": 101.395, "p95_ms": 93.741}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.913, "count": 27, "max_ms": 146.47, "p95_ms": 145.968}, "read_interval": {"avg_ms": 1152.14, "count": 26, "max_ms": 16023.613, "p95_ms": 11134.256}, "triton_wait": {"avg_ms": 79.624, "count": 27, "max_ms": 111.871, "p95_ms": 106.283}, "visualize": {"avg_ms": 4.137, "count": 27, "max_ms": 5.589, "p95_ms": 5.463}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5454.074, "limit_mb": 8796093022207.996}, "detail": {"pid": 12589, "restart_count": 26, "start_count": 27}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T14:58:52+09:00", "last_error_summary": "[h264 @ 0x55e82838d680] error while decoding MB 118 20, bytestream -35 [in#0/rtsp @ 0x55e827a8be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e827a8be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:58:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T14:58:55+09:00", "pid": 12589, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.676, "name": "ffmpeg", "pid": 12589, "rss_mb": 57.676, "state": "S (sleeping)", "threads": 1}, "restart_count": 26, "restart_count_per_hour": 13, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.973, "sample_count": 2003, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T14:58:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.45, "last_frame_pushed_at": "2026-07-06T14:58:53+09:00", "last_frame_read_age_sec": 4.549, "last_frame_read_at": "2026-07-06T14:58:52+09:00", "mjpeg": {"age_sec": 4.445, "exists": true, "mtime": "2026-07-06T14:58:53.033058+09:00", "mtime_epoch": 1783317533.0330584, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222743}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T14:58:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1331.855, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.639, "pushed": 0.639, "read": 0.639}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.981, "count": 14, "max_ms": 9.526, "p95_ms": 9.526}, "gstreamer_push": {"avg_ms": 1.613, "count": 14, "max_ms": 6.174, "p95_ms": 6.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.285, "count": 14, "max_ms": 79.708, "p95_ms": 79.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.553, "count": 14, "max_ms": 111.858, "p95_ms": 111.858}, "yolo11_640_fire_11class": {"avg_ms": 67.66, "count": 14, "max_ms": 86.43, "p95_ms": 86.43}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.941, "count": 14, "max_ms": 101.395, "p95_ms": 101.395}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.162, "count": 14, "max_ms": 124.205, "p95_ms": 124.205}, "read_interval": {"avg_ms": 2035.543, "count": 14, "max_ms": 16023.613, "p95_ms": 16023.613}, "triton_wait": {"avg_ms": 79.01, "count": 14, "max_ms": 111.871, "p95_ms": 111.871}, "visualize": {"avg_ms": 3.928, "count": 14, "max_ms": 5.09, "p95_ms": 5.09}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.912, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 617}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5487.977, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5600f8310780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5600f8310780] error while decoding MB 0 28, bytestream 7274 [h264 @ 0x5600f8df8940] cabac decode of qscale diff failed at 48 30 [h264 @ 0x5600f8df8940] error while decoding MB 48 30, bytestream 1364 [h264 @ 0x5600f83e3c80] left block unavailable for requested intra mode [h264 @ 0x5600f83e3c80] error while decoding MB 0 7, bytestream 8357 [h264 @ 0x5600f83e3c80] error while decoding MB 34 27, bytestream -8 [h2", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 123, "h264_decode_error_count_per_hour": 82, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:04:12+09:00", "last_error_summary": "[h264 @ 0x5600f8310780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5600f8310780] error while decoding MB 0 28, bytestream 7274 [h264 @ 0x5600f8df8940] cabac decode of qscale diff failed at 48 30 [h264 @ 0x5600f8df8940] error while decoding MB 48 30, bytestream 1364 [h264 @ 0x5600f83e3c80] left block unavailable for requested intra mode [h264 @ 0x5600f83e3c80] error while decoding MB 0 7, bytestream 8357 [h264 @ 0x5600f83e3c80] error while decoding MB 34 27, bytestream -8 [h2", "last_exit_at": "2026-07-06T14:54:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 602.043, "last_started_at": "2026-07-06T14:54:10+09:00", "pid": 12330, "process": {"exists": false, "pid": 12330}, "restart_count": 17, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.838, "sample_count": 2704, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74059, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:04:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.042, "last_frame_pushed_at": "2026-07-06T15:04:12+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T15:04:12+09:00", "mjpeg": {"age_sec": 0.024, "exists": true, "mtime": "2026-07-06T15:04:12.301479+09:00", "mtime_epoch": 1783317852.3014793, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188862}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:04:12+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1472.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.438, "pushed": 10.438, "read": 10.438}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.437, "count": 122, "max_ms": 5.0, "p95_ms": 4.165}, "gstreamer_push": {"avg_ms": 1.518, "count": 122, "max_ms": 6.795, "p95_ms": 2.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.944, "count": 122, "max_ms": 85.203, "p95_ms": 68.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.096, "count": 122, "max_ms": 113.396, "p95_ms": 98.132}, "yolo11_640_fire_11class": {"avg_ms": 59.008, "count": 122, "max_ms": 86.423, "p95_ms": 78.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.536, "count": 122, "max_ms": 98.788, "p95_ms": 87.799}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.443, "count": 122, "max_ms": 125.838, "p95_ms": 110.28}, "read_interval": {"avg_ms": 254.128, "count": 122, "max_ms": 19432.567, "p95_ms": 124.701}, "triton_wait": {"avg_ms": 71.109, "count": 122, "max_ms": 113.407, "p95_ms": 98.147}, "visualize": {"avg_ms": 3.78, "count": 122, "max_ms": 5.54, "p95_ms": 4.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.688, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 617}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5487.969, "limit_mb": 8796093022207.996}, "detail": {"actual_bytes": 0, "expected_bytes": 2764800}, "event_type": "invalid_frame_read_size", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 123, "h264_decode_error_count_per_hour": 74, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:04:12+09:00", "last_error_summary": "[h264 @ 0x5600f8310780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5600f8310780] error while decoding MB 0 28, bytestream 7274 [h264 @ 0x5600f8df8940] cabac decode of qscale diff failed at 48 30 [h264 @ 0x5600f8df8940] error while decoding MB 48 30, bytestream 1364 [h264 @ 0x5600f83e3c80] left block unavailable for requested intra mode [h264 @ 0x5600f83e3c80] error while decoding MB 0 7, bytestream 8357 [h264 @ 0x5600f83e3c80] error while decoding MB 34 27, bytestream -8 [h2", "last_exit_at": "2026-07-06T14:54:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 602.042, "last_started_at": "2026-07-06T14:54:10+09:00", "pid": 12330, "process": {"exists": false, "pid": 12330}, "restart_count": 17, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.838, "sample_count": 2704, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74059, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:04:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.041, "last_frame_pushed_at": "2026-07-06T15:04:12+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T15:04:12+09:00", "mjpeg": {"age_sec": 0.024, "exists": true, "mtime": "2026-07-06T15:04:12.301479+09:00", "mtime_epoch": 1783317852.3014793, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188862}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:04:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1472.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.434, "pushed": 10.434, "read": 10.434}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.437, "count": 122, "max_ms": 5.0, "p95_ms": 4.165}, "gstreamer_push": {"avg_ms": 1.518, "count": 122, "max_ms": 6.795, "p95_ms": 2.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.944, "count": 122, "max_ms": 85.203, "p95_ms": 68.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.096, "count": 122, "max_ms": 113.396, "p95_ms": 98.132}, "yolo11_640_fire_11class": {"avg_ms": 59.008, "count": 122, "max_ms": 86.423, "p95_ms": 78.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.536, "count": 122, "max_ms": 98.788, "p95_ms": 87.799}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.443, "count": 122, "max_ms": 125.838, "p95_ms": 110.28}, "read_interval": {"avg_ms": 254.128, "count": 122, "max_ms": 19432.567, "p95_ms": 124.701}, "triton_wait": {"avg_ms": 71.109, "count": 122, "max_ms": 113.407, "p95_ms": 98.147}, "visualize": {"avg_ms": 3.78, "count": 122, "max_ms": 5.54, "p95_ms": 4.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.692, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 617}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5487.781, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 123, "h264_decode_error_count_per_hour": 82, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:04:12+09:00", "last_error_summary": "[h264 @ 0x5600f8310780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5600f8310780] error while decoding MB 0 28, bytestream 7274 [h264 @ 0x5600f8df8940] cabac decode of qscale diff failed at 48 30 [h264 @ 0x5600f8df8940] error while decoding MB 48 30, bytestream 1364 [h264 @ 0x5600f83e3c80] left block unavailable for requested intra mode [h264 @ 0x5600f83e3c80] error while decoding MB 0 7, bytestream 8357 [h264 @ 0x5600f83e3c80] error while decoding MB 34 27, bytestream -8 [h2", "last_exit_at": "2026-07-06T14:54:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 602.065, "last_started_at": "2026-07-06T14:54:10+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 17, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.838, "sample_count": 2703, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74059, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:04:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-06T15:04:12+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T15:04:12+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-06T15:04:12.301479+09:00", "mtime_epoch": 1783317852.3014793, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188862}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:04:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1472.875, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.418, "pushed": 10.418, "read": 10.418}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.437, "count": 122, "max_ms": 5.0, "p95_ms": 4.165}, "gstreamer_push": {"avg_ms": 1.518, "count": 122, "max_ms": 6.795, "p95_ms": 2.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.944, "count": 122, "max_ms": 85.203, "p95_ms": 68.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.096, "count": 122, "max_ms": 113.396, "p95_ms": 98.132}, "yolo11_640_fire_11class": {"avg_ms": 59.008, "count": 122, "max_ms": 86.423, "p95_ms": 78.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.536, "count": 122, "max_ms": 98.788, "p95_ms": 87.799}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.443, "count": 122, "max_ms": 125.838, "p95_ms": 110.28}, "read_interval": {"avg_ms": 254.128, "count": 122, "max_ms": 19432.567, "p95_ms": 124.701}, "triton_wait": {"avg_ms": 71.109, "count": 122, "max_ms": 113.407, "p95_ms": 98.147}, "visualize": {"avg_ms": 3.78, "count": 122, "max_ms": 5.54, "p95_ms": 4.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.711, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 617}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5609.105, "limit_mb": 8796093022207.996}, "detail": {"pid": 12741, "restart_count": 18, "start_count": 19}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 123, "h264_decode_error_count_per_hour": 82, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:04:12+09:00", "last_error_summary": "[h264 @ 0x5600f8310780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5600f8310780] error while decoding MB 0 28, bytestream 7274 [h264 @ 0x5600f8df8940] cabac decode of qscale diff failed at 48 30 [h264 @ 0x5600f8df8940] error while decoding MB 48 30, bytestream 1364 [h264 @ 0x5600f83e3c80] left block unavailable for requested intra mode [h264 @ 0x5600f83e3c80] error while decoding MB 0 7, bytestream 8357 [h264 @ 0x5600f83e3c80] error while decoding MB 34 27, bytestream -8 [h2", "last_exit_at": "2026-07-06T14:54:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:04:14+09:00", "pid": 12741, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.648, "name": "ffmpeg", "pid": 12741, "rss_mb": 144.648, "state": "S (sleeping)", "threads": 106}, "restart_count": 18, "restart_count_per_hour": 9, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.838, "sample_count": 2657, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74059, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:04:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.666, "last_frame_pushed_at": "2026-07-06T15:04:12+09:00", "last_frame_read_age_sec": 4.741, "last_frame_read_at": "2026-07-06T15:04:12+09:00", "mjpeg": {"age_sec": 4.648, "exists": true, "mtime": "2026-07-06T15:04:12.301479+09:00", "mtime_epoch": 1783317852.3014793, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188862}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:04:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1472.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.479, "pushed": 7.479, "read": 7.479}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.437, "count": 122, "max_ms": 5.0, "p95_ms": 4.165}, "gstreamer_push": {"avg_ms": 1.518, "count": 122, "max_ms": 6.795, "p95_ms": 2.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.944, "count": 122, "max_ms": 85.203, "p95_ms": 68.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.096, "count": 122, "max_ms": 113.396, "p95_ms": 98.132}, "yolo11_640_fire_11class": {"avg_ms": 59.008, "count": 122, "max_ms": 86.423, "p95_ms": 78.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.536, "count": 122, "max_ms": 98.788, "p95_ms": 87.799}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.443, "count": 122, "max_ms": 125.838, "p95_ms": 110.28}, "read_interval": {"avg_ms": 254.128, "count": 122, "max_ms": 19432.567, "p95_ms": 124.701}, "triton_wait": {"avg_ms": 71.109, "count": 122, "max_ms": 113.407, "p95_ms": 98.147}, "visualize": {"avg_ms": 3.78, "count": 122, "max_ms": 5.54, "p95_ms": 4.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.312, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 531}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5426.727, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559a19b4e000] error while decoding MB 2 46, bytestream -11 [in#0/rtsp @ 0x559a19b06e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559a19b06e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:05:08+09:00", "last_error_summary": "[h264 @ 0x559a19b4e000] error while decoding MB 2 46, bytestream -11 [in#0/rtsp @ 0x559a19b06e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559a19b06e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T14:58:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 373.323, "last_started_at": "2026-07-06T14:58:55+09:00", "pid": 12589, "process": {"exists": false, "pid": 12589}, "restart_count": 26, "restart_count_per_hour": 12, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.957, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63799, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:05:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.051, "last_frame_pushed_at": "2026-07-06T15:05:08+09:00", "last_frame_read_age_sec": 0.043, "last_frame_read_at": "2026-07-06T15:05:08+09:00", "mjpeg": {"age_sec": 0.034, "exists": true, "mtime": "2026-07-06T15:05:08.765785+09:00", "mtime_epoch": 1783317908.7657852, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243243}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:05:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1336.949, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 50, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.668, "pushed": 1.668, "read": 1.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.151, "count": 49, "max_ms": 8.03, "p95_ms": 6.852}, "gstreamer_push": {"avg_ms": 1.349, "count": 50, "max_ms": 3.617, "p95_ms": 2.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.37, "count": 49, "max_ms": 84.604, "p95_ms": 81.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.846, "count": 49, "max_ms": 122.613, "p95_ms": 116.308}, "yolo11_640_fire_11class": {"avg_ms": 67.644, "count": 49, "max_ms": 86.958, "p95_ms": 83.749}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.657, "count": 49, "max_ms": 106.966, "p95_ms": 102.254}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.283, "count": 50, "max_ms": 174.098, "p95_ms": 155.749}, "read_interval": {"avg_ms": 601.362, "count": 50, "max_ms": 15152.544, "p95_ms": 186.322}, "triton_wait": {"avg_ms": 80.363, "count": 49, "max_ms": 122.625, "p95_ms": 116.321}, "visualize": {"avg_ms": 4.444, "count": 49, "max_ms": 5.983, "p95_ms": 5.413}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 531}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5426.512, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:05:08+09:00", "last_error_summary": "[h264 @ 0x559a19b4e000] error while decoding MB 2 46, bytestream -11 [in#0/rtsp @ 0x559a19b06e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559a19b06e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:05:08+09:00", "last_error_summary": "[h264 @ 0x559a19b4e000] error while decoding MB 2 46, bytestream -11 [in#0/rtsp @ 0x559a19b06e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559a19b06e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:05:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 373.389, "last_started_at": "2026-07-06T14:58:55+09:00", "pid": 12589, "process": {"exists": false, "pid": 12589}, "restart_count": 26, "restart_count_per_hour": 12, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.957, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63800, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:05:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T15:05:08+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T15:05:08+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T15:05:08.765785+09:00", "mtime_epoch": 1783317908.7657852, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243243}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:05:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1336.949, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.673, "pushed": 1.673, "read": 1.639}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.158, "count": 50, "max_ms": 8.03, "p95_ms": 6.852}, "gstreamer_push": {"avg_ms": 1.319, "count": 50, "max_ms": 3.617, "p95_ms": 1.731}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.542, "count": 50, "max_ms": 84.604, "p95_ms": 81.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.942, "count": 50, "max_ms": 122.613, "p95_ms": 116.308}, "yolo11_640_fire_11class": {"avg_ms": 67.646, "count": 50, "max_ms": 86.958, "p95_ms": 83.749}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.748, "count": 50, "max_ms": 106.966, "p95_ms": 102.254}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.56, "count": 50, "max_ms": 174.098, "p95_ms": 155.749}, "read_interval": {"avg_ms": 610.645, "count": 49, "max_ms": 15152.544, "p95_ms": 186.322}, "triton_wait": {"avg_ms": 80.429, "count": 50, "max_ms": 122.625, "p95_ms": 116.321}, "visualize": {"avg_ms": 4.445, "count": 50, "max_ms": 5.983, "p95_ms": 5.413}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 531}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5426.426, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:05:08+09:00", "last_error_summary": "[h264 @ 0x559a19b4e000] error while decoding MB 2 46, bytestream -11 [in#0/rtsp @ 0x559a19b06e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559a19b06e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:05:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 373.399, "last_started_at": "2026-07-06T14:58:55+09:00", "pid": 12589, "process": {"exists": false, "pid": 12589}, "restart_count": 26, "restart_count_per_hour": 12, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.957, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63800, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:05:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-06T15:05:08+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T15:05:08+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-06T15:05:08.765785+09:00", "mtime_epoch": 1783317908.7657852, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243243}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:05:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1336.949, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.672, "pushed": 1.672, "read": 1.638}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.158, "count": 50, "max_ms": 8.03, "p95_ms": 6.852}, "gstreamer_push": {"avg_ms": 1.319, "count": 50, "max_ms": 3.617, "p95_ms": 1.731}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.542, "count": 50, "max_ms": 84.604, "p95_ms": 81.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.942, "count": 50, "max_ms": 122.613, "p95_ms": 116.308}, "yolo11_640_fire_11class": {"avg_ms": 67.646, "count": 50, "max_ms": 86.958, "p95_ms": 83.749}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.748, "count": 50, "max_ms": 106.966, "p95_ms": 102.254}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.56, "count": 50, "max_ms": 174.098, "p95_ms": 155.749}, "read_interval": {"avg_ms": 610.645, "count": 49, "max_ms": 15152.544, "p95_ms": 186.322}, "triton_wait": {"avg_ms": 80.429, "count": 50, "max_ms": 122.625, "p95_ms": 116.321}, "visualize": {"avg_ms": 4.445, "count": 50, "max_ms": 5.983, "p95_ms": 5.413}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 531}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5608.48, "limit_mb": 8796093022207.996}, "detail": {"pid": 12848, "restart_count": 27, "start_count": 28}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:05:08+09:00", "last_error_summary": "[h264 @ 0x559a19b4e000] error while decoding MB 2 46, bytestream -11 [in#0/rtsp @ 0x559a19b06e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559a19b06e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:05:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:05:11+09:00", "pid": 12848, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.781, "name": "ffmpeg", "pid": 12848, "rss_mb": 209.781, "state": "S (sleeping)", "threads": 106}, "restart_count": 27, "restart_count_per_hour": 13, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.957, "sample_count": 2109, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63800, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:05:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.336, "last_frame_pushed_at": "2026-07-06T15:05:08+09:00", "last_frame_read_age_sec": 4.437, "last_frame_read_at": "2026-07-06T15:05:08+09:00", "mjpeg": {"age_sec": 4.316, "exists": true, "mtime": "2026-07-06T15:05:08.877786+09:00", "mtime_epoch": 1783317908.877786, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:05:13+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1336.949, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 15, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.534, "pushed": 0.534, "read": 0.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.238, "count": 16, "max_ms": 8.03, "p95_ms": 8.03}, "gstreamer_push": {"avg_ms": 1.389, "count": 16, "max_ms": 2.408, "p95_ms": 2.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.075, "count": 16, "max_ms": 81.507, "p95_ms": 81.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.53, "count": 16, "max_ms": 116.308, "p95_ms": 116.308}, "yolo11_640_fire_11class": {"avg_ms": 69.317, "count": 16, "max_ms": 86.958, "p95_ms": 86.958}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.811, "count": 16, "max_ms": 106.966, "p95_ms": 106.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.54, "count": 16, "max_ms": 140.0, "p95_ms": 140.0}, "read_interval": {"avg_ms": 1708.213, "count": 15, "max_ms": 15152.544, "p95_ms": 15152.544}, "triton_wait": {"avg_ms": 83.914, "count": 16, "max_ms": 116.321, "p95_ms": 116.321}, "visualize": {"avg_ms": 4.308, "count": 16, "max_ms": 5.983, "p95_ms": 5.983}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 537}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5429.473, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5631bd934fc0] error while decoding MB 108 29, bytestream -5 [in#0/rtsp @ 0x5631bd298e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5631bd298e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:07:12+09:00", "last_error_summary": "[h264 @ 0x5631bd934fc0] error while decoding MB 108 29, bytestream -5 [in#0/rtsp @ 0x5631bd298e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5631bd298e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:05:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.738, "last_started_at": "2026-07-06T15:05:11+09:00", "pid": 12848, "process": {"exists": false, "pid": 12848}, "restart_count": 27, "restart_count_per_hour": 13, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.412, "sample_count": 1910, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64558, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:07:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.052, "last_frame_pushed_at": "2026-07-06T15:07:12+09:00", "last_frame_read_age_sec": 0.042, "last_frame_read_at": "2026-07-06T15:07:12+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-06T15:07:12.898640+09:00", "mtime_epoch": 1783318032.8986404, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:07:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1325.91, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.682, "pushed": 0.682, "read": 0.739}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.253, "count": 12, "max_ms": 5.967, "p95_ms": 5.967}, "gstreamer_push": {"avg_ms": 1.733, "count": 12, "max_ms": 6.234, "p95_ms": 6.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.114, "count": 12, "max_ms": 81.414, "p95_ms": 81.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.726, "count": 12, "max_ms": 120.961, "p95_ms": 120.961}, "yolo11_640_fire_11class": {"avg_ms": 64.021, "count": 12, "max_ms": 84.44, "p95_ms": 84.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.13, "count": 12, "max_ms": 92.03, "p95_ms": 92.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.604, "count": 12, "max_ms": 143.581, "p95_ms": 143.581}, "read_interval": {"avg_ms": 2332.478, "count": 13, "max_ms": 16354.676, "p95_ms": 16354.676}, "triton_wait": {"avg_ms": 76.947, "count": 12, "max_ms": 120.973, "p95_ms": 120.973}, "visualize": {"avg_ms": 4.027, "count": 12, "max_ms": 5.73, "p95_ms": 5.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.601, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 537}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5436.32, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:07:12+09:00", "last_error_summary": "[h264 @ 0x5631bd934fc0] error while decoding MB 108 29, bytestream -5 [in#0/rtsp @ 0x5631bd298e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5631bd298e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:07:12+09:00", "last_error_summary": "[h264 @ 0x5631bd934fc0] error while decoding MB 108 29, bytestream -5 [in#0/rtsp @ 0x5631bd298e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5631bd298e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:07:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.806, "last_started_at": "2026-07-06T15:05:11+09:00", "pid": 12848, "process": {"exists": false, "pid": 12848}, "restart_count": 27, "restart_count_per_hour": 13, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.412, "sample_count": 1910, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64559, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:07:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T15:07:12+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T15:07:12+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T15:07:12.898640+09:00", "mtime_epoch": 1783318032.8986404, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:07:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1333.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.736, "pushed": 0.736, "read": 0.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.232, "count": 13, "max_ms": 5.967, "p95_ms": 5.967}, "gstreamer_push": {"avg_ms": 2.084, "count": 13, "max_ms": 6.294, "p95_ms": 6.294}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.828, "count": 13, "max_ms": 81.414, "p95_ms": 81.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.243, "count": 13, "max_ms": 120.961, "p95_ms": 120.961}, "yolo11_640_fire_11class": {"avg_ms": 64.13, "count": 13, "max_ms": 84.44, "p95_ms": 84.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.239, "count": 13, "max_ms": 92.03, "p95_ms": 92.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.778, "count": 13, "max_ms": 143.581, "p95_ms": 143.581}, "read_interval": {"avg_ms": 2332.478, "count": 13, "max_ms": 16354.676, "p95_ms": 16354.676}, "triton_wait": {"avg_ms": 77.448, "count": 13, "max_ms": 120.973, "p95_ms": 120.973}, "visualize": {"avg_ms": 4.009, "count": 13, "max_ms": 5.73, "p95_ms": 5.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.669, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 537}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5436.312, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:07:12+09:00", "last_error_summary": "[h264 @ 0x5631bd934fc0] error while decoding MB 108 29, bytestream -5 [in#0/rtsp @ 0x5631bd298e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5631bd298e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:07:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.813, "last_started_at": "2026-07-06T15:05:11+09:00", "pid": 12848, "process": {"exists": false, "pid": 12848}, "restart_count": 27, "restart_count_per_hour": 13, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.412, "sample_count": 1910, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64559, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:07:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T15:07:12+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-06T15:07:12+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T15:07:12.898640+09:00", "mtime_epoch": 1783318032.8986404, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:07:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1333.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.735, "pushed": 0.735, "read": 0.735}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.232, "count": 13, "max_ms": 5.967, "p95_ms": 5.967}, "gstreamer_push": {"avg_ms": 2.084, "count": 13, "max_ms": 6.294, "p95_ms": 6.294}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.828, "count": 13, "max_ms": 81.414, "p95_ms": 81.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.243, "count": 13, "max_ms": 120.961, "p95_ms": 120.961}, "yolo11_640_fire_11class": {"avg_ms": 64.13, "count": 13, "max_ms": 84.44, "p95_ms": 84.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.239, "count": 13, "max_ms": 92.03, "p95_ms": 92.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.778, "count": 13, "max_ms": 143.581, "p95_ms": 143.581}, "read_interval": {"avg_ms": 2332.478, "count": 13, "max_ms": 16354.676, "p95_ms": 16354.676}, "triton_wait": {"avg_ms": 77.448, "count": 13, "max_ms": 120.973, "p95_ms": 120.973}, "visualize": {"avg_ms": 4.009, "count": 13, "max_ms": 5.73, "p95_ms": 5.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.676, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 537}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5468.168, "limit_mb": 8796093022207.996}, "detail": {"pid": 13000, "restart_count": 28, "start_count": 29}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:07:12+09:00", "last_error_summary": "[h264 @ 0x5631bd934fc0] error while decoding MB 108 29, bytestream -5 [in#0/rtsp @ 0x5631bd298e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5631bd298e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:07:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:07:15+09:00", "pid": 13000, "process": {"exists": true, "fd_count": 4, "hwm_mb": 60.027, "name": "ffmpeg", "pid": 13000, "rss_mb": 60.027, "state": "R (running)", "threads": 13}, "restart_count": 28, "restart_count_per_hour": 14, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.875, "sample_count": 1867, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64559, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:07:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.653, "last_frame_pushed_at": "2026-07-06T15:07:12+09:00", "last_frame_read_age_sec": 4.759, "last_frame_read_at": "2026-07-06T15:07:12+09:00", "mjpeg": {"age_sec": 4.636, "exists": true, "mtime": "2026-07-06T15:07:13.010641+09:00", "mtime_epoch": 1783318033.010641, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236945}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:07:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1333.645, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.583, "pushed": 0.583, "read": 0.583}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.232, "count": 13, "max_ms": 5.967, "p95_ms": 5.967}, "gstreamer_push": {"avg_ms": 2.084, "count": 13, "max_ms": 6.294, "p95_ms": 6.294}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.828, "count": 13, "max_ms": 81.414, "p95_ms": 81.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.243, "count": 13, "max_ms": 120.961, "p95_ms": 120.961}, "yolo11_640_fire_11class": {"avg_ms": 64.13, "count": 13, "max_ms": 84.44, "p95_ms": 84.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.239, "count": 13, "max_ms": 92.03, "p95_ms": 92.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.778, "count": 13, "max_ms": 143.581, "p95_ms": 143.581}, "read_interval": {"avg_ms": 2332.478, "count": 13, "max_ms": 16354.676, "p95_ms": 16354.676}, "triton_wait": {"avg_ms": 77.448, "count": 13, "max_ms": 120.973, "p95_ms": 120.973}, "visualize": {"avg_ms": 4.009, "count": 13, "max_ms": 5.73, "p95_ms": 5.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.317, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 659}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5516.98, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560f086ccdc0] left block unavailable for requested intra mode [h264 @ 0x560f086ccdc0] error while decoding MB 0 10, bytestream 86188 [h264 @ 0x560f0859a180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f0859a180] error while decoding MB 0 17, bytestream 2299 [h264 @ 0x560f091ce240] error while decoding MB 60 13, bytestream -5 [h264 @ 0x560f086ccdc0] error while decoding MB 63 31, bytestream -6 [h264 @ 0x560f0929d000] cabac decode of qscale diff failed at 28 41 [h2", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 131, "h264_decode_error_count_per_hour": 78, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:13:38+09:00", "last_error_summary": "[h264 @ 0x560f086ccdc0] left block unavailable for requested intra mode [h264 @ 0x560f086ccdc0] error while decoding MB 0 10, bytestream 86188 [h264 @ 0x560f0859a180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f0859a180] error while decoding MB 0 17, bytestream 2299 [h264 @ 0x560f091ce240] error while decoding MB 60 13, bytestream -5 [h264 @ 0x560f086ccdc0] error while decoding MB 63 31, bytestream -6 [h264 @ 0x560f0929d000] cabac decode of qscale diff failed at 28 41 [h2", "last_exit_at": "2026-07-06T14:54:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 563.797, "last_started_at": "2026-07-06T15:04:14+09:00", "pid": 12741, "process": {"exists": false, "pid": 12741}, "restart_count": 18, "restart_count_per_hour": 8, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.328, "sample_count": 2686, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79184, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:13:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T15:13:38+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T15:13:38+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T15:13:38.702428+09:00", "mtime_epoch": 1783318418.7024276, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:13:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1477.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 176, "pushed_frames": 176, "read_frames": 176, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.88, "pushed": 5.88, "read": 5.88}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.298, "count": 176, "max_ms": 5.203, "p95_ms": 4.354}, "gstreamer_push": {"avg_ms": 1.416, "count": 176, "max_ms": 6.702, "p95_ms": 2.332}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.046, "count": 176, "max_ms": 85.012, "p95_ms": 70.185}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.591, "count": 176, "max_ms": 119.501, "p95_ms": 99.624}, "yolo11_640_fire_11class": {"avg_ms": 57.712, "count": 176, "max_ms": 85.084, "p95_ms": 78.601}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.645, "count": 176, "max_ms": 99.178, "p95_ms": 88.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.683, "count": 176, "max_ms": 132.504, "p95_ms": 111.472}, "read_interval": {"avg_ms": 170.267, "count": 176, "max_ms": 10124.811, "p95_ms": 132.077}, "triton_wait": {"avg_ms": 70.894, "count": 176, "max_ms": 119.514, "p95_ms": 99.637}, "visualize": {"avg_ms": 3.575, "count": 176, "max_ms": 5.045, "p95_ms": 4.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 659}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5516.277, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:13:38+09:00", "last_error_summary": "[h264 @ 0x560f086ccdc0] left block unavailable for requested intra mode [h264 @ 0x560f086ccdc0] error while decoding MB 0 10, bytestream 86188 [h264 @ 0x560f0859a180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f0859a180] error while decoding MB 0 17, bytestream 2299 [h264 @ 0x560f091ce240] error while decoding MB 60 13, bytestream -5 [h264 @ 0x560f086ccdc0] error while decoding MB 63 31, bytestream -6 [h264 @ 0x560f0929d000] cabac decode of qscale diff failed at 28 41 [h2", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 131, "h264_decode_error_count_per_hour": 78, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:13:38+09:00", "last_error_summary": "[h264 @ 0x560f086ccdc0] left block unavailable for requested intra mode [h264 @ 0x560f086ccdc0] error while decoding MB 0 10, bytestream 86188 [h264 @ 0x560f0859a180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f0859a180] error while decoding MB 0 17, bytestream 2299 [h264 @ 0x560f091ce240] error while decoding MB 60 13, bytestream -5 [h264 @ 0x560f086ccdc0] error while decoding MB 63 31, bytestream -6 [h264 @ 0x560f0929d000] cabac decode of qscale diff failed at 28 41 [h2", "last_exit_at": "2026-07-06T15:13:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 563.854, "last_started_at": "2026-07-06T15:04:14+09:00", "pid": 12741, "process": {"exists": false, "pid": 12741}, "restart_count": 18, "restart_count_per_hour": 8, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.328, "sample_count": 2687, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79185, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:13:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T15:13:38+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T15:13:38+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T15:13:38.702428+09:00", "mtime_epoch": 1783318418.7024276, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:13:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1477.816, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.902, "pushed": 5.902, "read": 5.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.298, "count": 177, "max_ms": 5.203, "p95_ms": 4.354}, "gstreamer_push": {"avg_ms": 1.415, "count": 177, "max_ms": 6.702, "p95_ms": 2.332}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.144, "count": 177, "max_ms": 85.012, "p95_ms": 70.185}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.685, "count": 177, "max_ms": 119.501, "p95_ms": 99.624}, "yolo11_640_fire_11class": {"avg_ms": 57.776, "count": 177, "max_ms": 85.084, "p95_ms": 78.601}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.763, "count": 177, "max_ms": 99.178, "p95_ms": 88.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.776, "count": 177, "max_ms": 132.504, "p95_ms": 111.472}, "read_interval": {"avg_ms": 170.267, "count": 176, "max_ms": 10124.811, "p95_ms": 132.077}, "triton_wait": {"avg_ms": 70.986, "count": 177, "max_ms": 119.514, "p95_ms": 99.637}, "visualize": {"avg_ms": 3.575, "count": 177, "max_ms": 5.045, "p95_ms": 4.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 659}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5516.215, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 131, "h264_decode_error_count_per_hour": 78, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:13:38+09:00", "last_error_summary": "[h264 @ 0x560f086ccdc0] left block unavailable for requested intra mode [h264 @ 0x560f086ccdc0] error while decoding MB 0 10, bytestream 86188 [h264 @ 0x560f0859a180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f0859a180] error while decoding MB 0 17, bytestream 2299 [h264 @ 0x560f091ce240] error while decoding MB 60 13, bytestream -5 [h264 @ 0x560f086ccdc0] error while decoding MB 63 31, bytestream -6 [h264 @ 0x560f0929d000] cabac decode of qscale diff failed at 28 41 [h2", "last_exit_at": "2026-07-06T15:13:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 563.862, "last_started_at": "2026-07-06T15:04:14+09:00", "pid": 12741, "process": {"exists": false, "pid": 12741}, "restart_count": 18, "restart_count_per_hour": 8, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.328, "sample_count": 2687, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79185, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:13:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T15:13:38+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T15:13:38+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T15:13:38.702428+09:00", "mtime_epoch": 1783318418.7024276, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:13:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1477.816, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.9, "pushed": 5.9, "read": 5.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.302, "count": 176, "max_ms": 5.203, "p95_ms": 4.354}, "gstreamer_push": {"avg_ms": 1.415, "count": 177, "max_ms": 6.702, "p95_ms": 2.332}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.106, "count": 176, "max_ms": 85.012, "p95_ms": 70.185}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.658, "count": 176, "max_ms": 119.501, "p95_ms": 99.624}, "yolo11_640_fire_11class": {"avg_ms": 57.757, "count": 176, "max_ms": 85.084, "p95_ms": 78.601}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.706, "count": 176, "max_ms": 99.178, "p95_ms": 88.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.776, "count": 177, "max_ms": 132.504, "p95_ms": 111.472}, "read_interval": {"avg_ms": 170.267, "count": 176, "max_ms": 10124.811, "p95_ms": 132.077}, "triton_wait": {"avg_ms": 70.961, "count": 176, "max_ms": 119.514, "p95_ms": 99.637}, "visualize": {"avg_ms": 3.581, "count": 176, "max_ms": 5.045, "p95_ms": 4.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 659}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5614.867, "limit_mb": 8796093022207.996}, "detail": {"pid": 13152, "restart_count": 19, "start_count": 20}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 131, "h264_decode_error_count_per_hour": 78, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:13:38+09:00", "last_error_summary": "[h264 @ 0x560f086ccdc0] left block unavailable for requested intra mode [h264 @ 0x560f086ccdc0] error while decoding MB 0 10, bytestream 86188 [h264 @ 0x560f0859a180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f0859a180] error while decoding MB 0 17, bytestream 2299 [h264 @ 0x560f091ce240] error while decoding MB 60 13, bytestream -5 [h264 @ 0x560f086ccdc0] error while decoding MB 63 31, bytestream -6 [h264 @ 0x560f0929d000] cabac decode of qscale diff failed at 28 41 [h2", "last_exit_at": "2026-07-06T15:13:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:13:41+09:00", "pid": 13152, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.418, "name": "ffmpeg", "pid": 13152, "rss_mb": 136.418, "state": "S (sleeping)", "threads": 106}, "restart_count": 19, "restart_count_per_hour": 9, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.328, "sample_count": 2642, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79185, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:13:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.906, "last_frame_pushed_at": "2026-07-06T15:13:38+09:00", "last_frame_read_age_sec": 5.007, "last_frame_read_at": "2026-07-06T15:13:38+09:00", "mjpeg": {"age_sec": 4.879, "exists": true, "mtime": "2026-07-06T15:13:38.822429+09:00", "mtime_epoch": 1783318418.822429, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213470}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:13:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1477.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.2, "pushed": 4.2, "read": 4.167}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.375, "count": 125, "max_ms": 5.203, "p95_ms": 4.159}, "gstreamer_push": {"avg_ms": 1.421, "count": 126, "max_ms": 6.702, "p95_ms": 1.481}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.585, "count": 125, "max_ms": 85.012, "p95_ms": 72.71}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.357, "count": 125, "max_ms": 119.501, "p95_ms": 98.0}, "yolo11_640_fire_11class": {"avg_ms": 58.131, "count": 125, "max_ms": 85.084, "p95_ms": 78.601}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.886, "count": 125, "max_ms": 99.178, "p95_ms": 88.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.828, "count": 126, "max_ms": 132.504, "p95_ms": 111.247}, "read_interval": {"avg_ms": 200.671, "count": 125, "max_ms": 10124.811, "p95_ms": 130.639}, "triton_wait": {"avg_ms": 70.624, "count": 125, "max_ms": 119.514, "p95_ms": 99.191}, "visualize": {"avg_ms": 3.711, "count": 125, "max_ms": 5.045, "p95_ms": 4.343}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5511.5, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564aa9434cc0] cabac decode of qscale diff failed at 50 17 [h264 @ 0x564aa9434cc0] error while decoding MB 50 17, bytestream 1017 [h264 @ 0x564aaa1c3d00] left block unavailable for requested intra mode [h264 @ 0x564aaa1c3d00] error while decoding MB 0 25, bytestream 4343 [h264 @ 0x564aa9e88300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa9e88300] error while decoding MB 0 17, bytestream 37862 [h264 @ 0x564aa9549200] cabac decode of qscale diff failed at 52 27 [h", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 139, "h264_decode_error_count_per_hour": 71, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:18:12+09:00", "last_error_summary": "[h264 @ 0x564aa9434cc0] cabac decode of qscale diff failed at 50 17 [h264 @ 0x564aa9434cc0] error while decoding MB 50 17, bytestream 1017 [h264 @ 0x564aaa1c3d00] left block unavailable for requested intra mode [h264 @ 0x564aaa1c3d00] error while decoding MB 0 25, bytestream 4343 [h264 @ 0x564aa9e88300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa9e88300] error while decoding MB 0 17, bytestream 37862 [h264 @ 0x564aa9549200] cabac decode of qscale diff failed at 52 27 [h", "last_exit_at": "2026-07-06T15:13:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 270.756, "last_started_at": "2026-07-06T15:13:41+09:00", "pid": 13152, "process": {"exists": false, "pid": 13152}, "restart_count": 19, "restart_count_per_hour": 8, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.973, "sample_count": 2374, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81421, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T15:18:12+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T15:18:12+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T15:18:12.405745+09:00", "mtime_epoch": 1783318692.405745, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:18:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1476.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.135, "pushed": 3.135, "read": 3.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.321, "count": 94, "max_ms": 4.974, "p95_ms": 4.016}, "gstreamer_push": {"avg_ms": 1.551, "count": 94, "max_ms": 3.954, "p95_ms": 3.693}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.223, "count": 94, "max_ms": 92.106, "p95_ms": 84.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.513, "count": 94, "max_ms": 117.694, "p95_ms": 113.073}, "yolo11_640_fire_11class": {"avg_ms": 73.792, "count": 94, "max_ms": 94.316, "p95_ms": 90.669}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.936, "count": 94, "max_ms": 106.829, "p95_ms": 99.618}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.295, "count": 94, "max_ms": 129.417, "p95_ms": 124.626}, "read_interval": {"avg_ms": 319.632, "count": 94, "max_ms": 19663.451, "p95_ms": 139.962}, "triton_wait": {"avg_ms": 86.269, "count": 94, "max_ms": 117.704, "p95_ms": 113.085}, "visualize": {"avg_ms": 3.623, "count": 94, "max_ms": 4.531, "p95_ms": 4.076}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5510.988, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:18:12+09:00", "last_error_summary": "[h264 @ 0x564aa9434cc0] cabac decode of qscale diff failed at 50 17 [h264 @ 0x564aa9434cc0] error while decoding MB 50 17, bytestream 1017 [h264 @ 0x564aaa1c3d00] left block unavailable for requested intra mode [h264 @ 0x564aaa1c3d00] error while decoding MB 0 25, bytestream 4343 [h264 @ 0x564aa9e88300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa9e88300] error while decoding MB 0 17, bytestream 37862 [h264 @ 0x564aa9549200] cabac decode of qscale diff failed at 52 27 [h", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 139, "h264_decode_error_count_per_hour": 71, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:18:12+09:00", "last_error_summary": "[h264 @ 0x564aa9434cc0] cabac decode of qscale diff failed at 50 17 [h264 @ 0x564aa9434cc0] error while decoding MB 50 17, bytestream 1017 [h264 @ 0x564aaa1c3d00] left block unavailable for requested intra mode [h264 @ 0x564aaa1c3d00] error while decoding MB 0 25, bytestream 4343 [h264 @ 0x564aa9e88300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa9e88300] error while decoding MB 0 17, bytestream 37862 [h264 @ 0x564aa9549200] cabac decode of qscale diff failed at 52 27 [h", "last_exit_at": "2026-07-06T15:18:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 270.818, "last_started_at": "2026-07-06T15:13:41+09:00", "pid": 13152, "process": {"exists": false, "pid": 13152}, "restart_count": 19, "restart_count_per_hour": 8, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.973, "sample_count": 2374, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81422, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T15:18:12+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-06T15:18:12+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T15:18:12.405745+09:00", "mtime_epoch": 1783318692.405745, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:18:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1476.102, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.141, "pushed": 3.141, "read": 3.108}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.32, "count": 94, "max_ms": 4.974, "p95_ms": 4.016}, "gstreamer_push": {"avg_ms": 1.551, "count": 94, "max_ms": 3.954, "p95_ms": 3.693}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.914, "count": 94, "max_ms": 92.106, "p95_ms": 82.632}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.428, "count": 94, "max_ms": 117.694, "p95_ms": 113.073}, "yolo11_640_fire_11class": {"avg_ms": 73.681, "count": 94, "max_ms": 94.316, "p95_ms": 90.669}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.769, "count": 94, "max_ms": 106.829, "p95_ms": 96.939}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.205, "count": 94, "max_ms": 129.417, "p95_ms": 124.626}, "read_interval": {"avg_ms": 321.703, "count": 93, "max_ms": 19663.451, "p95_ms": 139.962}, "triton_wait": {"avg_ms": 86.184, "count": 94, "max_ms": 117.704, "p95_ms": 113.085}, "visualize": {"avg_ms": 3.623, "count": 94, "max_ms": 4.531, "p95_ms": 4.076}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5511.078, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 139, "h264_decode_error_count_per_hour": 71, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:18:12+09:00", "last_error_summary": "[h264 @ 0x564aa9434cc0] cabac decode of qscale diff failed at 50 17 [h264 @ 0x564aa9434cc0] error while decoding MB 50 17, bytestream 1017 [h264 @ 0x564aaa1c3d00] left block unavailable for requested intra mode [h264 @ 0x564aaa1c3d00] error while decoding MB 0 25, bytestream 4343 [h264 @ 0x564aa9e88300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa9e88300] error while decoding MB 0 17, bytestream 37862 [h264 @ 0x564aa9549200] cabac decode of qscale diff failed at 52 27 [h", "last_exit_at": "2026-07-06T15:18:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 270.825, "last_started_at": "2026-07-06T15:13:41+09:00", "pid": 13152, "process": {"exists": false, "pid": 13152}, "restart_count": 19, "restart_count_per_hour": 8, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.973, "sample_count": 2374, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81422, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T15:18:12+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-06T15:18:12+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T15:18:12.405745+09:00", "mtime_epoch": 1783318692.405745, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:18:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1476.102, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.14, "pushed": 3.14, "read": 3.107}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.32, "count": 94, "max_ms": 4.974, "p95_ms": 4.016}, "gstreamer_push": {"avg_ms": 1.551, "count": 94, "max_ms": 3.954, "p95_ms": 3.693}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.914, "count": 94, "max_ms": 92.106, "p95_ms": 82.632}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.428, "count": 94, "max_ms": 117.694, "p95_ms": 113.073}, "yolo11_640_fire_11class": {"avg_ms": 73.681, "count": 94, "max_ms": 94.316, "p95_ms": 90.669}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.769, "count": 94, "max_ms": 106.829, "p95_ms": 96.939}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.205, "count": 94, "max_ms": 129.417, "p95_ms": 124.626}, "read_interval": {"avg_ms": 321.703, "count": 93, "max_ms": 19663.451, "p95_ms": 139.962}, "triton_wait": {"avg_ms": 86.184, "count": 94, "max_ms": 117.704, "p95_ms": 113.085}, "visualize": {"avg_ms": 3.623, "count": 94, "max_ms": 4.531, "p95_ms": 4.076}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5607.961, "limit_mb": 8796093022207.996}, "detail": {"pid": 13259, "restart_count": 20, "start_count": 21}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 139, "h264_decode_error_count_per_hour": 71, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:18:12+09:00", "last_error_summary": "[h264 @ 0x564aa9434cc0] cabac decode of qscale diff failed at 50 17 [h264 @ 0x564aa9434cc0] error while decoding MB 50 17, bytestream 1017 [h264 @ 0x564aaa1c3d00] left block unavailable for requested intra mode [h264 @ 0x564aaa1c3d00] error while decoding MB 0 25, bytestream 4343 [h264 @ 0x564aa9e88300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa9e88300] error while decoding MB 0 17, bytestream 37862 [h264 @ 0x564aa9549200] cabac decode of qscale diff failed at 52 27 [h", "last_exit_at": "2026-07-06T15:18:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:18:15+09:00", "pid": 13259, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.414, "name": "ffmpeg", "pid": 13259, "rss_mb": 139.414, "state": "S (sleeping)", "threads": 106}, "restart_count": 20, "restart_count_per_hour": 9, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.216, "sample_count": 2326, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81422, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.616, "last_frame_pushed_at": "2026-07-06T15:18:12+09:00", "last_frame_read_age_sec": 4.724, "last_frame_read_at": "2026-07-06T15:18:12+09:00", "mjpeg": {"age_sec": 4.598, "exists": true, "mtime": "2026-07-06T15:18:12.529747+09:00", "mtime_epoch": 1783318692.5297468, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:18:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1476.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.769, "pushed": 1.769, "read": 1.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.3, "count": 53, "max_ms": 4.108, "p95_ms": 3.86}, "gstreamer_push": {"avg_ms": 1.468, "count": 53, "max_ms": 3.954, "p95_ms": 3.427}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.805, "count": 53, "max_ms": 84.751, "p95_ms": 80.948}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.116, "count": 53, "max_ms": 115.994, "p95_ms": 113.073}, "yolo11_640_fire_11class": {"avg_ms": 73.048, "count": 53, "max_ms": 92.331, "p95_ms": 88.132}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.947, "count": 53, "max_ms": 103.437, "p95_ms": 96.422}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.106, "count": 53, "max_ms": 128.43, "p95_ms": 124.808}, "read_interval": {"avg_ms": 487.637, "count": 52, "max_ms": 19663.451, "p95_ms": 140.992}, "triton_wait": {"avg_ms": 86.094, "count": 53, "max_ms": 116.008, "p95_ms": 113.085}, "visualize": {"avg_ms": 3.619, "count": 53, "max_ms": 3.838, "p95_ms": 3.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 584}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5439.867, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55edd74dce00] error while decoding MB 101 32, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:18:45+09:00", "last_error_summary": "[h264 @ 0x55edd74dce00] error while decoding MB 101 32, bytestream -7", "last_exit_at": "2026-07-06T15:07:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 689.856, "last_started_at": "2026-07-06T15:07:15+09:00", "pid": 13000, "process": {"exists": false, "pid": 13000}, "restart_count": 28, "restart_count_per_hour": 13, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.002, "sample_count": 2526, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70155, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:18:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T15:18:45+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T15:18:45+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-06T15:18:45.442168+09:00", "mtime_epoch": 1783318725.4421675, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224621}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:18:45+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1340.152, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 255, "pushed_frames": 255, "read_frames": 255, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.501, "pushed": 8.501, "read": 8.501}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.156, "count": 254, "max_ms": 8.413, "p95_ms": 6.214}, "gstreamer_push": {"avg_ms": 1.54, "count": 255, "max_ms": 8.011, "p95_ms": 3.698}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.907, "count": 254, "max_ms": 87.629, "p95_ms": 80.379}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.851, "count": 254, "max_ms": 125.177, "p95_ms": 115.095}, "yolo11_640_fire_11class": {"avg_ms": 73.635, "count": 254, "max_ms": 99.464, "p95_ms": 86.537}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.085, "count": 254, "max_ms": 103.133, "p95_ms": 96.398}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.291, "count": 255, "max_ms": 149.478, "p95_ms": 132.485}, "read_interval": {"avg_ms": 117.738, "count": 255, "max_ms": 159.636, "p95_ms": 146.989}, "triton_wait": {"avg_ms": 88.413, "count": 254, "max_ms": 125.191, "p95_ms": 115.108}, "visualize": {"avg_ms": 3.821, "count": 254, "max_ms": 5.878, "p95_ms": 5.047}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 584}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5439.297, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:18:45+09:00", "last_error_summary": "[h264 @ 0x55edd74dce00] error while decoding MB 101 32, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:18:45+09:00", "last_error_summary": "[h264 @ 0x55edd74dce00] error while decoding MB 101 32, bytestream -7", "last_exit_at": "2026-07-06T15:18:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 689.911, "last_started_at": "2026-07-06T15:07:15+09:00", "pid": 13000, "process": {"exists": false, "pid": 13000}, "restart_count": 28, "restart_count_per_hour": 13, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.002, "sample_count": 2527, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70156, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:18:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T15:18:45+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-06T15:18:45+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T15:18:45.442168+09:00", "mtime_epoch": 1783318725.4421675, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224621}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:18:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1340.152, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 255, "pushed_frames": 255, "read_frames": 254, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.51, "pushed": 8.51, "read": 8.476}, "hints": ["read_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.155, "count": 255, "max_ms": 8.413, "p95_ms": 6.214}, "gstreamer_push": {"avg_ms": 1.54, "count": 255, "max_ms": 8.011, "p95_ms": 3.698}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.94, "count": 255, "max_ms": 87.629, "p95_ms": 80.379}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.895, "count": 255, "max_ms": 125.177, "p95_ms": 115.095}, "yolo11_640_fire_11class": {"avg_ms": 73.7, "count": 255, "max_ms": 99.464, "p95_ms": 86.944}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.131, "count": 255, "max_ms": 103.133, "p95_ms": 96.398}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.361, "count": 255, "max_ms": 149.478, "p95_ms": 132.485}, "read_interval": {"avg_ms": 117.781, "count": 254, "max_ms": 159.636, "p95_ms": 146.989}, "triton_wait": {"avg_ms": 88.455, "count": 255, "max_ms": 125.191, "p95_ms": 115.108}, "visualize": {"avg_ms": 3.82, "count": 255, "max_ms": 5.878, "p95_ms": 5.047}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 584}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5439.348, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:18:45+09:00", "last_error_summary": "[h264 @ 0x55edd74dce00] error while decoding MB 101 32, bytestream -7", "last_exit_at": "2026-07-06T15:18:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 689.922, "last_started_at": "2026-07-06T15:07:15+09:00", "pid": 13000, "process": {"exists": false, "pid": 13000}, "restart_count": 28, "restart_count_per_hour": 13, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.002, "sample_count": 2527, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70156, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:18:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-06T15:18:45+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-06T15:18:45+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-06T15:18:45.442168+09:00", "mtime_epoch": 1783318725.4421675, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224621}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:18:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1340.152, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 255, "pushed_frames": 255, "read_frames": 254, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.507, "pushed": 8.507, "read": 8.473}, "hints": ["read_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.155, "count": 255, "max_ms": 8.413, "p95_ms": 6.214}, "gstreamer_push": {"avg_ms": 1.54, "count": 255, "max_ms": 8.011, "p95_ms": 3.698}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.94, "count": 255, "max_ms": 87.629, "p95_ms": 80.379}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.895, "count": 255, "max_ms": 125.177, "p95_ms": 115.095}, "yolo11_640_fire_11class": {"avg_ms": 73.7, "count": 255, "max_ms": 99.464, "p95_ms": 86.944}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.131, "count": 255, "max_ms": 103.133, "p95_ms": 96.398}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.361, "count": 255, "max_ms": 149.478, "p95_ms": 132.485}, "read_interval": {"avg_ms": 117.781, "count": 254, "max_ms": 159.636, "p95_ms": 146.989}, "triton_wait": {"avg_ms": 88.455, "count": 255, "max_ms": 125.191, "p95_ms": 115.108}, "visualize": {"avg_ms": 3.82, "count": 255, "max_ms": 5.878, "p95_ms": 5.047}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 584}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5465.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 13366, "restart_count": 29, "start_count": 30}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:18:45+09:00", "last_error_summary": "[h264 @ 0x55edd74dce00] error while decoding MB 101 32, bytestream -7", "last_exit_at": "2026-07-06T15:18:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:18:48+09:00", "pid": 13366, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.109, "name": "ffmpeg", "pid": 13366, "rss_mb": 57.109, "state": "S (sleeping)", "threads": 1}, "restart_count": 29, "restart_count_per_hour": 14, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.633, "sample_count": 2491, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70156, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:18:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.883, "last_frame_pushed_at": "2026-07-06T15:18:45+09:00", "last_frame_read_age_sec": 4.996, "last_frame_read_at": "2026-07-06T15:18:45+09:00", "mjpeg": {"age_sec": 4.866, "exists": true, "mtime": "2026-07-06T15:18:45.566169+09:00", "mtime_epoch": 1783318725.566169, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:18:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1340.152, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 211, "pushed_frames": 211, "read_frames": 210, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.046, "pushed": 7.046, "read": 7.013}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.219, "count": 211, "max_ms": 8.413, "p95_ms": 6.231}, "gstreamer_push": {"avg_ms": 1.526, "count": 211, "max_ms": 8.011, "p95_ms": 3.69}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.255, "count": 211, "max_ms": 82.433, "p95_ms": 80.379}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 88.913, "count": 211, "max_ms": 125.177, "p95_ms": 115.391}, "yolo11_640_fire_11class": {"avg_ms": 74.012, "count": 211, "max_ms": 99.464, "p95_ms": 86.537}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.561, "count": 211, "max_ms": 100.185, "p95_ms": 96.351}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.903, "count": 211, "max_ms": 149.478, "p95_ms": 133.377}, "read_interval": {"avg_ms": 119.318, "count": 210, "max_ms": 159.636, "p95_ms": 148.387}, "triton_wait": {"avg_ms": 89.331, "count": 211, "max_ms": 125.191, "p95_ms": 115.404}, "visualize": {"avg_ms": 3.851, "count": 211, "max_ms": 5.878, "p95_ms": 5.124}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5509.559, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555f01402440] error while decoding MB 68 18, bytestream -12 [h264 @ 0x555f00945d40] left block unavailable for requested intra mode [h264 @ 0x555f00945d40] error while decoding MB 0 26, bytestream 689 [h264 @ 0x555f014d12c0] left block unavailable for requested intra mode [h264 @ 0x555f014d12c0] error while decoding MB 0 4, bytestream 378 [h264 @ 0x555f01264740] left block unavailable for requested intra mode [h264 @ 0x555f01264740] error while decoding MB 0 37, bytestream 4975 [in#0/r", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 69, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:21:58+09:00", "last_error_summary": "[h264 @ 0x555f01402440] error while decoding MB 68 18, bytestream -12 [h264 @ 0x555f00945d40] left block unavailable for requested intra mode [h264 @ 0x555f00945d40] error while decoding MB 0 26, bytestream 689 [h264 @ 0x555f014d12c0] left block unavailable for requested intra mode [h264 @ 0x555f014d12c0] error while decoding MB 0 4, bytestream 378 [h264 @ 0x555f01264740] left block unavailable for requested intra mode [h264 @ 0x555f01264740] error while decoding MB 0 37, bytestream 4975 [in#0/r", "last_exit_at": "2026-07-06T15:18:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 223.795, "last_started_at": "2026-07-06T15:18:15+09:00", "pid": 13259, "process": {"exists": false, "pid": 13259}, "restart_count": 20, "restart_count_per_hour": 8, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.991, "sample_count": 2394, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T15:21:58+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T15:21:58+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T15:21:58.864709+09:00", "mtime_epoch": 1783318918.8647087, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196457}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:21:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1477.297, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 175, "select_timeouts": 10, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.843, "pushed": 5.843, "read": 5.843}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.404, "count": 175, "max_ms": 5.359, "p95_ms": 4.325}, "gstreamer_push": {"avg_ms": 1.522, "count": 175, "max_ms": 6.001, "p95_ms": 3.655}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.075, "count": 175, "max_ms": 88.843, "p95_ms": 80.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.571, "count": 175, "max_ms": 122.432, "p95_ms": 107.818}, "yolo11_640_fire_11class": {"avg_ms": 66.602, "count": 175, "max_ms": 92.678, "p95_ms": 88.913}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.188, "count": 175, "max_ms": 107.48, "p95_ms": 100.347}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.089, "count": 175, "max_ms": 134.827, "p95_ms": 119.721}, "read_interval": {"avg_ms": 171.385, "count": 175, "max_ms": 9600.481, "p95_ms": 138.698}, "triton_wait": {"avg_ms": 79.925, "count": 175, "max_ms": 122.444, "p95_ms": 107.829}, "visualize": {"avg_ms": 3.685, "count": 175, "max_ms": 4.752, "p95_ms": 4.207}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5508.652, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:21:58+09:00", "last_error_summary": "[h264 @ 0x555f01402440] error while decoding MB 68 18, bytestream -12 [h264 @ 0x555f00945d40] left block unavailable for requested intra mode [h264 @ 0x555f00945d40] error while decoding MB 0 26, bytestream 689 [h264 @ 0x555f014d12c0] left block unavailable for requested intra mode [h264 @ 0x555f014d12c0] error while decoding MB 0 4, bytestream 378 [h264 @ 0x555f01264740] left block unavailable for requested intra mode [h264 @ 0x555f01264740] error while decoding MB 0 37, bytestream 4975 [in#0/r", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 69, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:21:58+09:00", "last_error_summary": "[h264 @ 0x555f01402440] error while decoding MB 68 18, bytestream -12 [h264 @ 0x555f00945d40] left block unavailable for requested intra mode [h264 @ 0x555f00945d40] error while decoding MB 0 26, bytestream 689 [h264 @ 0x555f014d12c0] left block unavailable for requested intra mode [h264 @ 0x555f014d12c0] error while decoding MB 0 4, bytestream 378 [h264 @ 0x555f01264740] left block unavailable for requested intra mode [h264 @ 0x555f01264740] error while decoding MB 0 37, bytestream 4975 [in#0/r", "last_exit_at": "2026-07-06T15:21:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 223.861, "last_started_at": "2026-07-06T15:18:15+09:00", "pid": 13259, "process": {"exists": false, "pid": 13259}, "restart_count": 20, "restart_count_per_hour": 8, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.991, "sample_count": 2395, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T15:21:58+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T15:21:58+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T15:21:58.864709+09:00", "mtime_epoch": 1783318918.8647087, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196457}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:21:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1477.297, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 175, "select_timeouts": 10, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.833, "pushed": 5.833, "read": 5.833}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.403, "count": 175, "max_ms": 5.359, "p95_ms": 4.325}, "gstreamer_push": {"avg_ms": 1.524, "count": 175, "max_ms": 6.001, "p95_ms": 3.655}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.104, "count": 175, "max_ms": 88.843, "p95_ms": 80.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.639, "count": 175, "max_ms": 122.432, "p95_ms": 107.818}, "yolo11_640_fire_11class": {"avg_ms": 66.681, "count": 175, "max_ms": 92.678, "p95_ms": 89.006}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.334, "count": 175, "max_ms": 107.48, "p95_ms": 100.494}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.161, "count": 175, "max_ms": 134.827, "p95_ms": 119.721}, "read_interval": {"avg_ms": 171.385, "count": 175, "max_ms": 9600.481, "p95_ms": 138.698}, "triton_wait": {"avg_ms": 79.993, "count": 175, "max_ms": 122.444, "p95_ms": 107.829}, "visualize": {"avg_ms": 3.687, "count": 175, "max_ms": 4.752, "p95_ms": 4.207}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5508.617, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 69, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:21:58+09:00", "last_error_summary": "[h264 @ 0x555f01402440] error while decoding MB 68 18, bytestream -12 [h264 @ 0x555f00945d40] left block unavailable for requested intra mode [h264 @ 0x555f00945d40] error while decoding MB 0 26, bytestream 689 [h264 @ 0x555f014d12c0] left block unavailable for requested intra mode [h264 @ 0x555f014d12c0] error while decoding MB 0 4, bytestream 378 [h264 @ 0x555f01264740] left block unavailable for requested intra mode [h264 @ 0x555f01264740] error while decoding MB 0 37, bytestream 4975 [in#0/r", "last_exit_at": "2026-07-06T15:21:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 223.869, "last_started_at": "2026-07-06T15:18:15+09:00", "pid": 13259, "process": {"exists": false, "pid": 13259}, "restart_count": 20, "restart_count_per_hour": 8, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.991, "sample_count": 2395, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T15:21:58+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T15:21:58+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-06T15:21:58.864709+09:00", "mtime_epoch": 1783318918.8647087, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196457}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:21:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1477.297, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 10, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.85, "pushed": 5.85, "read": 5.816}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.403, "count": 175, "max_ms": 5.359, "p95_ms": 4.325}, "gstreamer_push": {"avg_ms": 1.524, "count": 175, "max_ms": 6.001, "p95_ms": 3.655}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.104, "count": 175, "max_ms": 88.843, "p95_ms": 80.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.639, "count": 175, "max_ms": 122.432, "p95_ms": 107.818}, "yolo11_640_fire_11class": {"avg_ms": 66.681, "count": 175, "max_ms": 92.678, "p95_ms": 89.006}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.334, "count": 175, "max_ms": 107.48, "p95_ms": 100.494}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.161, "count": 175, "max_ms": 134.827, "p95_ms": 119.721}, "read_interval": {"avg_ms": 171.674, "count": 174, "max_ms": 9600.481, "p95_ms": 138.698}, "triton_wait": {"avg_ms": 79.993, "count": 175, "max_ms": 122.444, "p95_ms": 107.829}, "visualize": {"avg_ms": 3.687, "count": 175, "max_ms": 4.752, "p95_ms": 4.207}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5624.074, "limit_mb": 8796093022207.996}, "detail": {"pid": 13518, "restart_count": 21, "start_count": 22}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 69, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:21:58+09:00", "last_error_summary": "[h264 @ 0x555f01402440] error while decoding MB 68 18, bytestream -12 [h264 @ 0x555f00945d40] left block unavailable for requested intra mode [h264 @ 0x555f00945d40] error while decoding MB 0 26, bytestream 689 [h264 @ 0x555f014d12c0] left block unavailable for requested intra mode [h264 @ 0x555f014d12c0] error while decoding MB 0 4, bytestream 378 [h264 @ 0x555f01264740] left block unavailable for requested intra mode [h264 @ 0x555f01264740] error while decoding MB 0 37, bytestream 4975 [in#0/r", "last_exit_at": "2026-07-06T15:22:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:22:01+09:00", "pid": 13518, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.887, "name": "ffmpeg", "pid": 13518, "rss_mb": 138.887, "state": "S (sleeping)", "threads": 106}, "restart_count": 21, "restart_count_per_hour": 9, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.125, "sample_count": 2361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.167, "last_frame_pushed_at": "2026-07-06T15:21:58+09:00", "last_frame_read_age_sec": 4.289, "last_frame_read_at": "2026-07-06T15:21:58+09:00", "mjpeg": {"age_sec": 4.152, "exists": true, "mtime": "2026-07-06T15:21:58.996710+09:00", "mtime_epoch": 1783318918.9967103, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197598}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:22:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1477.297, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 10, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.472, "pushed": 4.472, "read": 4.438}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.421, "count": 134, "max_ms": 5.359, "p95_ms": 4.583}, "gstreamer_push": {"avg_ms": 1.452, "count": 134, "max_ms": 4.44, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.669, "count": 134, "max_ms": 88.843, "p95_ms": 81.013}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.337, "count": 134, "max_ms": 122.432, "p95_ms": 112.069}, "yolo11_640_fire_11class": {"avg_ms": 68.198, "count": 134, "max_ms": 92.678, "p95_ms": 90.349}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.536, "count": 134, "max_ms": 107.48, "p95_ms": 101.851}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.023, "count": 134, "max_ms": 134.827, "p95_ms": 124.655}, "read_interval": {"avg_ms": 193.556, "count": 133, "max_ms": 9600.481, "p95_ms": 140.188}, "triton_wait": {"avg_ms": 81.795, "count": 134, "max_ms": 122.444, "p95_ms": 112.081}, "visualize": {"avg_ms": 3.7, "count": 134, "max_ms": 4.752, "p95_ms": 4.216}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 612}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5457.75, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557643efec80] error while decoding MB 29 10, bytestream -5 [in#0/rtsp @ 0x557643eede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643eede00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:26:04+09:00", "last_error_summary": "[h264 @ 0x557643efec80] error while decoding MB 29 10, bytestream -5 [in#0/rtsp @ 0x557643eede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643eede00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:18:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 436.208, "last_started_at": "2026-07-06T15:18:48+09:00", "pid": 13366, "process": {"exists": false, "pid": 13366}, "restart_count": 29, "restart_count_per_hour": 11, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.067, "sample_count": 2191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73445, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:26:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T15:26:04+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T15:26:04+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T15:26:04.584062+09:00", "mtime_epoch": 1783319164.5840616, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232373}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:26:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1341.605, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.769, "pushed": 2.769, "read": 2.769}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.159, "count": 82, "max_ms": 8.947, "p95_ms": 8.055}, "gstreamer_push": {"avg_ms": 1.234, "count": 83, "max_ms": 3.434, "p95_ms": 2.018}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.586, "count": 82, "max_ms": 85.281, "p95_ms": 74.147}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.465, "count": 82, "max_ms": 122.976, "p95_ms": 107.603}, "yolo11_640_fire_11class": {"avg_ms": 63.698, "count": 82, "max_ms": 93.945, "p95_ms": 85.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.185, "count": 82, "max_ms": 99.149, "p95_ms": 93.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.627, "count": 83, "max_ms": 186.755, "p95_ms": 160.771}, "read_interval": {"avg_ms": 361.741, "count": 83, "max_ms": 10166.227, "p95_ms": 172.984}, "triton_wait": {"avg_ms": 73.132, "count": 82, "max_ms": 122.99, "p95_ms": 107.616}, "visualize": {"avg_ms": 4.982, "count": 82, "max_ms": 6.233, "p95_ms": 5.978}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 612}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5460.984, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:26:04+09:00", "last_error_summary": "[h264 @ 0x557643efec80] error while decoding MB 29 10, bytestream -5 [in#0/rtsp @ 0x557643eede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643eede00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:26:04+09:00", "last_error_summary": "[h264 @ 0x557643efec80] error while decoding MB 29 10, bytestream -5 [in#0/rtsp @ 0x557643eede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643eede00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:26:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 436.269, "last_started_at": "2026-07-06T15:18:48+09:00", "pid": 13366, "process": {"exists": false, "pid": 13366}, "restart_count": 29, "restart_count_per_hour": 11, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.067, "sample_count": 2192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73446, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:26:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T15:26:04+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-06T15:26:04+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T15:26:04.584062+09:00", "mtime_epoch": 1783319164.5840616, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232373}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:26:04+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1341.605, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.776, "pushed": 2.776, "read": 2.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.163, "count": 83, "max_ms": 8.947, "p95_ms": 8.055}, "gstreamer_push": {"avg_ms": 1.235, "count": 83, "max_ms": 3.434, "p95_ms": 2.018}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.448, "count": 83, "max_ms": 85.281, "p95_ms": 74.147}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.274, "count": 83, "max_ms": 122.976, "p95_ms": 107.603}, "yolo11_640_fire_11class": {"avg_ms": 63.603, "count": 83, "max_ms": 93.945, "p95_ms": 85.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.072, "count": 83, "max_ms": 99.149, "p95_ms": 93.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.715, "count": 83, "max_ms": 186.755, "p95_ms": 160.771}, "read_interval": {"avg_ms": 364.683, "count": 82, "max_ms": 10166.227, "p95_ms": 172.984}, "triton_wait": {"avg_ms": 72.947, "count": 83, "max_ms": 122.99, "p95_ms": 107.616}, "visualize": {"avg_ms": 5.041, "count": 83, "max_ms": 9.836, "p95_ms": 6.156}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 612}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5460.695, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:26:04+09:00", "last_error_summary": "[h264 @ 0x557643efec80] error while decoding MB 29 10, bytestream -5 [in#0/rtsp @ 0x557643eede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643eede00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:26:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 436.274, "last_started_at": "2026-07-06T15:18:48+09:00", "pid": 13366, "process": {"exists": false, "pid": 13366}, "restart_count": 29, "restart_count_per_hour": 11, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.067, "sample_count": 2192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73446, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:26:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T15:26:04+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T15:26:04+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T15:26:04.584062+09:00", "mtime_epoch": 1783319164.5840616, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232373}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:26:04+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1341.605, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.776, "pushed": 2.776, "read": 2.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.163, "count": 83, "max_ms": 8.947, "p95_ms": 8.055}, "gstreamer_push": {"avg_ms": 1.235, "count": 83, "max_ms": 3.434, "p95_ms": 2.018}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.448, "count": 83, "max_ms": 85.281, "p95_ms": 74.147}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.274, "count": 83, "max_ms": 122.976, "p95_ms": 107.603}, "yolo11_640_fire_11class": {"avg_ms": 63.603, "count": 83, "max_ms": 93.945, "p95_ms": 85.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.072, "count": 83, "max_ms": 99.149, "p95_ms": 93.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.715, "count": 83, "max_ms": 186.755, "p95_ms": 160.771}, "read_interval": {"avg_ms": 364.683, "count": 82, "max_ms": 10166.227, "p95_ms": 172.984}, "triton_wait": {"avg_ms": 72.947, "count": 83, "max_ms": 122.99, "p95_ms": 107.616}, "visualize": {"avg_ms": 5.041, "count": 83, "max_ms": 9.836, "p95_ms": 6.156}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 612}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5512.418, "limit_mb": 8796093022207.996}, "detail": {"pid": 13625, "restart_count": 30, "start_count": 31}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:26:04+09:00", "last_error_summary": "[h264 @ 0x557643efec80] error while decoding MB 29 10, bytestream -5 [in#0/rtsp @ 0x557643eede00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643eede00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:26:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:26:07+09:00", "pid": 13625, "process": {"exists": true, "fd_count": 5, "hwm_mb": 94.113, "name": "ffmpeg", "pid": 13625, "rss_mb": 94.113, "state": "S (sleeping)", "threads": 19}, "restart_count": 30, "restart_count_per_hour": 12, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.116, "sample_count": 2150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73446, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:26:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.999, "last_frame_pushed_at": "2026-07-06T15:26:04+09:00", "last_frame_read_age_sec": 5.119, "last_frame_read_at": "2026-07-06T15:26:04+09:00", "mjpeg": {"age_sec": 4.982, "exists": true, "mtime": "2026-07-06T15:26:04.712063+09:00", "mtime_epoch": 1783319164.7120633, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231828}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:26:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1341.605, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 47, "pushed_frames": 48, "read_frames": 47, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.567, "pushed": 1.6, "read": 1.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.869, "count": 47, "max_ms": 8.274, "p95_ms": 7.897}, "gstreamer_push": {"avg_ms": 1.273, "count": 48, "max_ms": 3.434, "p95_ms": 2.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.533, "count": 47, "max_ms": 78.278, "p95_ms": 72.587}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.961, "count": 47, "max_ms": 110.819, "p95_ms": 101.846}, "yolo11_640_fire_11class": {"avg_ms": 63.188, "count": 47, "max_ms": 85.555, "p95_ms": 83.127}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.825, "count": 47, "max_ms": 96.984, "p95_ms": 93.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.62, "count": 47, "max_ms": 163.397, "p95_ms": 154.383}, "read_interval": {"avg_ms": 531.674, "count": 47, "max_ms": 10166.227, "p95_ms": 172.984}, "triton_wait": {"avg_ms": 70.769, "count": 47, "max_ms": 110.833, "p95_ms": 101.867}, "visualize": {"avg_ms": 4.899, "count": 47, "max_ms": 9.836, "p95_ms": 5.84}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 737}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5516.949, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557f8f9ca600] cabac decode of qscale diff failed at 48 44 [h264 @ 0x557f8f9ca600] error while decoding MB 48 44, bytestream 96943 [h264 @ 0x557f8f9f28c0] left block unavailable for requested intra mode [h264 @ 0x557f8f9f28c0] error while decoding MB 0 17, bytestream 46981 [h264 @ 0x557f904b6280] error while decoding MB 17 40, bytestream 17 [h264 @ 0x557f90318580] error while decoding MB 60 29, bytestream -5 [in#0/rtsp @ 0x557f8f89ce00] Error during demuxing: Connection timed out [in#0/", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 70, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:31:13+09:00", "last_error_summary": "[h264 @ 0x557f8f9ca600] cabac decode of qscale diff failed at 48 44 [h264 @ 0x557f8f9ca600] error while decoding MB 48 44, bytestream 96943 [h264 @ 0x557f8f9f28c0] left block unavailable for requested intra mode [h264 @ 0x557f8f9f28c0] error while decoding MB 0 17, bytestream 46981 [h264 @ 0x557f904b6280] error while decoding MB 17 40, bytestream 17 [h264 @ 0x557f90318580] error while decoding MB 60 29, bytestream -5 [in#0/rtsp @ 0x557f8f89ce00] Error during demuxing: Connection timed out [in#0/", "last_exit_at": "2026-07-06T15:22:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 552.809, "last_started_at": "2026-07-06T15:22:01+09:00", "pid": 13518, "process": {"exists": false, "pid": 13518}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.135, "sample_count": 2658, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 88469, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:31:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-06T15:31:13+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-06T15:31:13+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-06T15:31:13.904406+09:00", "mtime_epoch": 1783319473.9044056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196382}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:31:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1479.684, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 107, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.579, "pushed": 3.579, "read": 3.579}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.267, "count": 107, "max_ms": 5.418, "p95_ms": 4.254}, "gstreamer_push": {"avg_ms": 1.377, "count": 107, "max_ms": 3.948, "p95_ms": 1.594}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.254, "count": 107, "max_ms": 86.297, "p95_ms": 76.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.539, "count": 107, "max_ms": 118.29, "p95_ms": 106.982}, "yolo11_640_fire_11class": {"avg_ms": 62.393, "count": 107, "max_ms": 95.703, "p95_ms": 82.375}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.532, "count": 107, "max_ms": 99.69, "p95_ms": 93.793}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.549, "count": 107, "max_ms": 131.213, "p95_ms": 119.267}, "read_interval": {"avg_ms": 280.081, "count": 107, "max_ms": 10124.305, "p95_ms": 137.932}, "triton_wait": {"avg_ms": 75.701, "count": 107, "max_ms": 118.302, "p95_ms": 106.995}, "visualize": {"avg_ms": 3.561, "count": 107, "max_ms": 5.592, "p95_ms": 4.528}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 737}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5516.668, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:31:13+09:00", "last_error_summary": "[h264 @ 0x557f8f9ca600] cabac decode of qscale diff failed at 48 44 [h264 @ 0x557f8f9ca600] error while decoding MB 48 44, bytestream 96943 [h264 @ 0x557f8f9f28c0] left block unavailable for requested intra mode [h264 @ 0x557f8f9f28c0] error while decoding MB 0 17, bytestream 46981 [h264 @ 0x557f904b6280] error while decoding MB 17 40, bytestream 17 [h264 @ 0x557f90318580] error while decoding MB 60 29, bytestream -5 [in#0/rtsp @ 0x557f8f89ce00] Error during demuxing: Connection timed out [in#0/", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 70, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:31:13+09:00", "last_error_summary": "[h264 @ 0x557f8f9ca600] cabac decode of qscale diff failed at 48 44 [h264 @ 0x557f8f9ca600] error while decoding MB 48 44, bytestream 96943 [h264 @ 0x557f8f9f28c0] left block unavailable for requested intra mode [h264 @ 0x557f8f9f28c0] error while decoding MB 0 17, bytestream 46981 [h264 @ 0x557f904b6280] error while decoding MB 17 40, bytestream 17 [h264 @ 0x557f90318580] error while decoding MB 60 29, bytestream -5 [in#0/rtsp @ 0x557f8f89ce00] Error during demuxing: Connection timed out [in#0/", "last_exit_at": "2026-07-06T15:31:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 552.834, "last_started_at": "2026-07-06T15:22:01+09:00", "pid": 13518, "process": {"exists": false, "pid": 13518}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.135, "sample_count": 2659, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 88470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:31:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T15:31:13+09:00", "last_frame_read_age_sec": 0.083, "last_frame_read_at": "2026-07-06T15:31:13+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-06T15:31:13.904406+09:00", "mtime_epoch": 1783319473.9044056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196382}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:31:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1479.684, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.61, "pushed": 3.61, "read": 3.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.311, "count": 108, "max_ms": 7.992, "p95_ms": 4.564}, "gstreamer_push": {"avg_ms": 1.377, "count": 108, "max_ms": 3.948, "p95_ms": 1.594}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.079, "count": 108, "max_ms": 86.297, "p95_ms": 76.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.343, "count": 108, "max_ms": 118.29, "p95_ms": 106.982}, "yolo11_640_fire_11class": {"avg_ms": 62.254, "count": 108, "max_ms": 95.703, "p95_ms": 82.375}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.361, "count": 108, "max_ms": 99.69, "p95_ms": 93.793}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.426, "count": 108, "max_ms": 131.213, "p95_ms": 119.267}, "read_interval": {"avg_ms": 280.081, "count": 107, "max_ms": 10124.305, "p95_ms": 137.932}, "triton_wait": {"avg_ms": 75.503, "count": 108, "max_ms": 118.302, "p95_ms": 106.995}, "visualize": {"avg_ms": 3.59, "count": 108, "max_ms": 6.692, "p95_ms": 4.651}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 737}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5516.461, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 70, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:31:13+09:00", "last_error_summary": "[h264 @ 0x557f8f9ca600] cabac decode of qscale diff failed at 48 44 [h264 @ 0x557f8f9ca600] error while decoding MB 48 44, bytestream 96943 [h264 @ 0x557f8f9f28c0] left block unavailable for requested intra mode [h264 @ 0x557f8f9f28c0] error while decoding MB 0 17, bytestream 46981 [h264 @ 0x557f904b6280] error while decoding MB 17 40, bytestream 17 [h264 @ 0x557f90318580] error while decoding MB 60 29, bytestream -5 [in#0/rtsp @ 0x557f8f89ce00] Error during demuxing: Connection timed out [in#0/", "last_exit_at": "2026-07-06T15:31:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 552.842, "last_started_at": "2026-07-06T15:22:01+09:00", "pid": 13518, "process": {"exists": false, "pid": 13518}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.135, "sample_count": 2659, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 88470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:31:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T15:31:13+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-06T15:31:13+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-06T15:31:13.904406+09:00", "mtime_epoch": 1783319473.9044056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196382}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:31:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1479.684, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.609, "pushed": 3.609, "read": 3.575}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.311, "count": 108, "max_ms": 7.992, "p95_ms": 4.564}, "gstreamer_push": {"avg_ms": 1.377, "count": 108, "max_ms": 3.948, "p95_ms": 1.594}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.079, "count": 108, "max_ms": 86.297, "p95_ms": 76.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.343, "count": 108, "max_ms": 118.29, "p95_ms": 106.982}, "yolo11_640_fire_11class": {"avg_ms": 62.254, "count": 108, "max_ms": 95.703, "p95_ms": 82.375}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.361, "count": 108, "max_ms": 99.69, "p95_ms": 93.793}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.426, "count": 108, "max_ms": 131.213, "p95_ms": 119.267}, "read_interval": {"avg_ms": 280.081, "count": 107, "max_ms": 10124.305, "p95_ms": 137.932}, "triton_wait": {"avg_ms": 75.503, "count": 108, "max_ms": 118.302, "p95_ms": 106.995}, "visualize": {"avg_ms": 3.59, "count": 108, "max_ms": 6.692, "p95_ms": 4.651}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 737}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5631.082, "limit_mb": 8796093022207.996}, "detail": {"pid": 13777, "restart_count": 22, "start_count": 23}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 70, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:31:13+09:00", "last_error_summary": "[h264 @ 0x557f8f9ca600] cabac decode of qscale diff failed at 48 44 [h264 @ 0x557f8f9ca600] error while decoding MB 48 44, bytestream 96943 [h264 @ 0x557f8f9f28c0] left block unavailable for requested intra mode [h264 @ 0x557f8f9f28c0] error while decoding MB 0 17, bytestream 46981 [h264 @ 0x557f904b6280] error while decoding MB 17 40, bytestream 17 [h264 @ 0x557f90318580] error while decoding MB 60 29, bytestream -5 [in#0/rtsp @ 0x557f8f89ce00] Error during demuxing: Connection timed out [in#0/", "last_exit_at": "2026-07-06T15:31:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:31:16+09:00", "pid": 13777, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.918, "name": "ffmpeg", "pid": 13777, "rss_mb": 138.918, "state": "S (sleeping)", "threads": 106}, "restart_count": 22, "restart_count_per_hour": 9, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.135, "sample_count": 2619, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 88470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:31:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.205, "last_frame_pushed_at": "2026-07-06T15:31:13+09:00", "last_frame_read_age_sec": 4.281, "last_frame_read_at": "2026-07-06T15:31:13+09:00", "mjpeg": {"age_sec": 4.191, "exists": true, "mtime": "2026-07-06T15:31:13.988407+09:00", "mtime_epoch": 1783319473.9884067, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196597}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:31:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1479.684, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.238, "pushed": 2.238, "read": 2.205}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.293, "count": 67, "max_ms": 7.992, "p95_ms": 3.894}, "gstreamer_push": {"avg_ms": 1.442, "count": 67, "max_ms": 3.948, "p95_ms": 3.566}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.409, "count": 67, "max_ms": 80.548, "p95_ms": 77.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.953, "count": 67, "max_ms": 118.29, "p95_ms": 105.765}, "yolo11_640_fire_11class": {"avg_ms": 61.092, "count": 67, "max_ms": 95.703, "p95_ms": 83.691}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.621, "count": 67, "max_ms": 99.69, "p95_ms": 94.638}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.869, "count": 67, "max_ms": 130.638, "p95_ms": 118.192}, "read_interval": {"avg_ms": 389.98, "count": 66, "max_ms": 10124.305, "p95_ms": 143.031}, "triton_wait": {"avg_ms": 73.978, "count": 67, "max_ms": 118.302, "p95_ms": 105.778}, "visualize": {"avg_ms": 3.607, "count": 67, "max_ms": 6.692, "p95_ms": 4.37}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 641}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5477.816, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5594d4e40840] error while decoding MB 62 41, bytestream -5 [in#0/rtsp @ 0x5594d4132e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594d4132e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:33:45+09:00", "last_error_summary": "[h264 @ 0x5594d4e40840] error while decoding MB 62 41, bytestream -5 [in#0/rtsp @ 0x5594d4132e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594d4132e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:26:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 457.626, "last_started_at": "2026-07-06T15:26:07+09:00", "pid": 13625, "process": {"exists": false, "pid": 13625}, "restart_count": 30, "restart_count_per_hour": 9, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.463, "sample_count": 2272, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77003, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:33:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.089, "last_frame_pushed_at": "2026-07-06T15:33:45+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-06T15:33:45+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-06T15:33:45.251460+09:00", "mtime_epoch": 1783319625.2514598, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 197989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:33:45+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1346.281, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.135, "pushed": 3.135, "read": 3.168}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.439, "count": 94, "max_ms": 8.678, "p95_ms": 6.088}, "gstreamer_push": {"avg_ms": 1.353, "count": 94, "max_ms": 6.042, "p95_ms": 1.431}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.536, "count": 94, "max_ms": 86.858, "p95_ms": 77.943}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.317, "count": 94, "max_ms": 115.389, "p95_ms": 106.586}, "yolo11_640_fire_11class": {"avg_ms": 67.763, "count": 94, "max_ms": 91.131, "p95_ms": 86.567}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.923, "count": 94, "max_ms": 104.159, "p95_ms": 100.823}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.837, "count": 94, "max_ms": 164.954, "p95_ms": 150.284}, "read_interval": {"avg_ms": 315.909, "count": 95, "max_ms": 19034.718, "p95_ms": 162.211}, "triton_wait": {"avg_ms": 82.633, "count": 94, "max_ms": 115.402, "p95_ms": 106.599}, "visualize": {"avg_ms": 3.943, "count": 94, "max_ms": 5.862, "p95_ms": 5.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 641}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5477.266, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:33:45+09:00", "last_error_summary": "[h264 @ 0x5594d4e40840] error while decoding MB 62 41, bytestream -5 [in#0/rtsp @ 0x5594d4132e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594d4132e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:33:45+09:00", "last_error_summary": "[h264 @ 0x5594d4e40840] error while decoding MB 62 41, bytestream -5 [in#0/rtsp @ 0x5594d4132e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594d4132e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:33:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 457.67, "last_started_at": "2026-07-06T15:26:07+09:00", "pid": 13625, "process": {"exists": false, "pid": 13625}, "restart_count": 30, "restart_count_per_hour": 9, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.463, "sample_count": 2272, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77004, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:33:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T15:33:45+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-06T15:33:45+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T15:33:45.251460+09:00", "mtime_epoch": 1783319625.2514598, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 197989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:33:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1346.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.175, "pushed": 3.175, "read": 3.142}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.434, "count": 95, "max_ms": 8.678, "p95_ms": 6.088}, "gstreamer_push": {"avg_ms": 1.353, "count": 95, "max_ms": 6.042, "p95_ms": 1.431}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.74, "count": 95, "max_ms": 86.858, "p95_ms": 78.493}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.537, "count": 95, "max_ms": 115.389, "p95_ms": 106.586}, "yolo11_640_fire_11class": {"avg_ms": 67.977, "count": 95, "max_ms": 91.131, "p95_ms": 86.611}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.161, "count": 95, "max_ms": 104.159, "p95_ms": 100.823}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.949, "count": 95, "max_ms": 164.954, "p95_ms": 150.284}, "read_interval": {"avg_ms": 318.151, "count": 94, "max_ms": 19034.718, "p95_ms": 162.211}, "triton_wait": {"avg_ms": 82.849, "count": 95, "max_ms": 115.402, "p95_ms": 106.599}, "visualize": {"avg_ms": 3.939, "count": 95, "max_ms": 5.862, "p95_ms": 5.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 641}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5477.328, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:33:45+09:00", "last_error_summary": "[h264 @ 0x5594d4e40840] error while decoding MB 62 41, bytestream -5 [in#0/rtsp @ 0x5594d4132e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594d4132e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:33:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 457.678, "last_started_at": "2026-07-06T15:26:07+09:00", "pid": 13625, "process": {"exists": false, "pid": 13625}, "restart_count": 30, "restart_count_per_hour": 9, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.463, "sample_count": 2272, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77004, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:33:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T15:33:45+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T15:33:45+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T15:33:45.251460+09:00", "mtime_epoch": 1783319625.2514598, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 197989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:33:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1346.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.174, "pushed": 3.174, "read": 3.141}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.434, "count": 95, "max_ms": 8.678, "p95_ms": 6.088}, "gstreamer_push": {"avg_ms": 1.353, "count": 95, "max_ms": 6.042, "p95_ms": 1.431}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.74, "count": 95, "max_ms": 86.858, "p95_ms": 78.493}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.537, "count": 95, "max_ms": 115.389, "p95_ms": 106.586}, "yolo11_640_fire_11class": {"avg_ms": 67.977, "count": 95, "max_ms": 91.131, "p95_ms": 86.611}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.161, "count": 95, "max_ms": 104.159, "p95_ms": 100.823}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.949, "count": 95, "max_ms": 164.954, "p95_ms": 150.284}, "read_interval": {"avg_ms": 318.151, "count": 94, "max_ms": 19034.718, "p95_ms": 162.211}, "triton_wait": {"avg_ms": 82.849, "count": 95, "max_ms": 115.402, "p95_ms": 106.599}, "visualize": {"avg_ms": 3.939, "count": 95, "max_ms": 5.862, "p95_ms": 5.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 641}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5535.391, "limit_mb": 8796093022207.996}, "detail": {"pid": 13884, "restart_count": 31, "start_count": 32}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:33:45+09:00", "last_error_summary": "[h264 @ 0x5594d4e40840] error while decoding MB 62 41, bytestream -5 [in#0/rtsp @ 0x5594d4132e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594d4132e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:33:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:33:47+09:00", "pid": 13884, "process": {"exists": true, "fd_count": 5, "hwm_mb": 93.848, "name": "ffmpeg", "pid": 13884, "rss_mb": 93.848, "state": "S (sleeping)", "threads": 19}, "restart_count": 31, "restart_count_per_hour": 10, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.463, "sample_count": 2237, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77004, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:33:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.369, "last_frame_pushed_at": "2026-07-06T15:33:45+09:00", "last_frame_read_age_sec": 4.487, "last_frame_read_at": "2026-07-06T15:33:45+09:00", "mjpeg": {"age_sec": 4.35, "exists": true, "mtime": "2026-07-06T15:33:45.375464+09:00", "mtime_epoch": 1783319625.3754642, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 197566}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:33:49+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1346.285, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.004, "pushed": 2.004, "read": 1.97}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.293, "count": 60, "max_ms": 8.678, "p95_ms": 6.088}, "gstreamer_push": {"avg_ms": 1.409, "count": 60, "max_ms": 6.042, "p95_ms": 2.35}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.725, "count": 60, "max_ms": 86.858, "p95_ms": 79.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.615, "count": 60, "max_ms": 115.389, "p95_ms": 104.62}, "yolo11_640_fire_11class": {"avg_ms": 66.789, "count": 60, "max_ms": 88.107, "p95_ms": 86.567}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.575, "count": 60, "max_ms": 104.159, "p95_ms": 100.823}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.069, "count": 60, "max_ms": 164.954, "p95_ms": 136.516}, "read_interval": {"avg_ms": 433.452, "count": 59, "max_ms": 19034.718, "p95_ms": 169.516}, "triton_wait": {"avg_ms": 81.042, "count": 60, "max_ms": 115.402, "p95_ms": 104.631}, "visualize": {"avg_ms": 3.827, "count": 60, "max_ms": 5.862, "p95_ms": 5.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 649}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5479.027, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562b2b904a00] error while decoding MB 84 44, bytestream -7 [in#0/rtsp @ 0x562b2b82ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b2b82ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:36:16+09:00", "last_error_summary": "[h264 @ 0x562b2b904a00] error while decoding MB 84 44, bytestream -7 [in#0/rtsp @ 0x562b2b82ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b2b82ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:33:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.015, "last_started_at": "2026-07-06T15:33:47+09:00", "pid": 13884, "process": {"exists": false, "pid": 13884}, "restart_count": 31, "restart_count_per_hour": 10, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.978, "sample_count": 2064, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77975, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:36:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T15:36:16+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-06T15:36:16+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T15:36:16.684432+09:00", "mtime_epoch": 1783319776.6844316, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 251011}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:36:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1343.516, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 65, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.17, "pushed": 2.17, "read": 2.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.191, "count": 65, "max_ms": 9.945, "p95_ms": 9.141}, "gstreamer_push": {"avg_ms": 1.298, "count": 65, "max_ms": 3.874, "p95_ms": 2.556}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.39, "count": 65, "max_ms": 84.007, "p95_ms": 79.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.506, "count": 65, "max_ms": 117.478, "p95_ms": 109.068}, "yolo11_640_fire_11class": {"avg_ms": 64.637, "count": 65, "max_ms": 90.552, "p95_ms": 82.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.769, "count": 65, "max_ms": 102.658, "p95_ms": 96.542}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.855, "count": 65, "max_ms": 172.543, "p95_ms": 163.479}, "read_interval": {"avg_ms": 462.06, "count": 65, "max_ms": 12373.692, "p95_ms": 182.091}, "triton_wait": {"avg_ms": 79.695, "count": 65, "max_ms": 117.49, "p95_ms": 109.08}, "visualize": {"avg_ms": 5.07, "count": 65, "max_ms": 7.287, "p95_ms": 6.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 649}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5477.906, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:36:16+09:00", "last_error_summary": "[h264 @ 0x562b2b904a00] error while decoding MB 84 44, bytestream -7 [in#0/rtsp @ 0x562b2b82ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b2b82ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:36:16+09:00", "last_error_summary": "[h264 @ 0x562b2b904a00] error while decoding MB 84 44, bytestream -7 [in#0/rtsp @ 0x562b2b82ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b2b82ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:36:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.084, "last_started_at": "2026-07-06T15:33:47+09:00", "pid": 13884, "process": {"exists": false, "pid": 13884}, "restart_count": 31, "restart_count_per_hour": 10, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.978, "sample_count": 2065, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:36:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T15:36:16+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-06T15:36:16+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-06T15:36:16.684432+09:00", "mtime_epoch": 1783319776.6844316, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 251011}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:36:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1343.516, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.175, "pushed": 2.175, "read": 2.141}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.281, "count": 65, "max_ms": 12.463, "p95_ms": 9.434}, "gstreamer_push": {"avg_ms": 1.296, "count": 65, "max_ms": 3.874, "p95_ms": 2.556}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.299, "count": 65, "max_ms": 84.007, "p95_ms": 79.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.974, "count": 65, "max_ms": 117.478, "p95_ms": 109.068}, "yolo11_640_fire_11class": {"avg_ms": 64.361, "count": 65, "max_ms": 90.552, "p95_ms": 82.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.364, "count": 65, "max_ms": 102.658, "p95_ms": 96.542}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.556, "count": 65, "max_ms": 172.543, "p95_ms": 163.479}, "read_interval": {"avg_ms": 466.78, "count": 64, "max_ms": 12373.692, "p95_ms": 182.091}, "triton_wait": {"avg_ms": 79.205, "count": 65, "max_ms": 117.49, "p95_ms": 109.08}, "visualize": {"avg_ms": 5.103, "count": 65, "max_ms": 7.691, "p95_ms": 6.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 649}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5483.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:36:16+09:00", "last_error_summary": "[h264 @ 0x562b2b904a00] error while decoding MB 84 44, bytestream -7 [in#0/rtsp @ 0x562b2b82ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b2b82ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:36:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.09, "last_started_at": "2026-07-06T15:33:47+09:00", "pid": 13884, "process": {"exists": false, "pid": 13884}, "restart_count": 31, "restart_count_per_hour": 10, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.978, "sample_count": 2065, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:36:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T15:36:16+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-06T15:36:16+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T15:36:16.684432+09:00", "mtime_epoch": 1783319776.6844316, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 251011}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:36:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1343.516, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.174, "pushed": 2.174, "read": 2.141}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.281, "count": 65, "max_ms": 12.463, "p95_ms": 9.434}, "gstreamer_push": {"avg_ms": 1.296, "count": 65, "max_ms": 3.874, "p95_ms": 2.556}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.299, "count": 65, "max_ms": 84.007, "p95_ms": 79.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.974, "count": 65, "max_ms": 117.478, "p95_ms": 109.068}, "yolo11_640_fire_11class": {"avg_ms": 64.361, "count": 65, "max_ms": 90.552, "p95_ms": 82.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.364, "count": 65, "max_ms": 102.658, "p95_ms": 96.542}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.556, "count": 65, "max_ms": 172.543, "p95_ms": 163.479}, "read_interval": {"avg_ms": 466.78, "count": 64, "max_ms": 12373.692, "p95_ms": 182.091}, "triton_wait": {"avg_ms": 79.205, "count": 65, "max_ms": 117.49, "p95_ms": 109.08}, "visualize": {"avg_ms": 5.103, "count": 65, "max_ms": 7.691, "p95_ms": 6.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 649}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5643.121, "limit_mb": 8796093022207.996}, "detail": {"pid": 14036, "restart_count": 32, "start_count": 33}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:36:16+09:00", "last_error_summary": "[h264 @ 0x562b2b904a00] error while decoding MB 84 44, bytestream -7 [in#0/rtsp @ 0x562b2b82ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b2b82ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:36:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:36:19+09:00", "pid": 14036, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.488, "name": "ffmpeg", "pid": 14036, "rss_mb": 194.488, "state": "S (sleeping)", "threads": 106}, "restart_count": 32, "restart_count_per_hour": 11, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.136, "sample_count": 2032, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:36:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.495, "last_frame_pushed_at": "2026-07-06T15:36:16+09:00", "last_frame_read_age_sec": 4.627, "last_frame_read_at": "2026-07-06T15:36:16+09:00", "mjpeg": {"age_sec": 4.475, "exists": true, "mtime": "2026-07-06T15:36:16.824436+09:00", "mtime_epoch": 1783319776.8244357, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250613}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:36:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1343.516, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.134, "pushed": 1.134, "read": 1.101}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.99, "count": 33, "max_ms": 12.463, "p95_ms": 9.945}, "gstreamer_push": {"avg_ms": 1.318, "count": 34, "max_ms": 3.874, "p95_ms": 3.383}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.466, "count": 33, "max_ms": 84.007, "p95_ms": 80.795}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.594, "count": 33, "max_ms": 117.478, "p95_ms": 115.261}, "yolo11_640_fire_11class": {"avg_ms": 68.44, "count": 33, "max_ms": 90.552, "p95_ms": 87.734}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.651, "count": 33, "max_ms": 102.658, "p95_ms": 97.102}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.183, "count": 34, "max_ms": 172.543, "p95_ms": 169.77}, "read_interval": {"avg_ms": 772.474, "count": 33, "max_ms": 12373.692, "p95_ms": 8919.475}, "triton_wait": {"avg_ms": 81.951, "count": 33, "max_ms": 117.49, "p95_ms": 115.273}, "visualize": {"avg_ms": 4.861, "count": 33, "max_ms": 7.691, "p95_ms": 6.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 792}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5539.809, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559cf84ac540] error while decoding MB 14 25, bytestream -8 [h264 @ 0x559cf90603c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559cf90603c0] error while decoding MB 0 34, bytestream 3453 [h264 @ 0x559cf84c0700] top block unavailable for requested intra mode [h264 @ 0x559cf84c0700] error while decoding MB 43 0, bytestream 5302 [h264 @ 0x559cf8501580] cabac decode of qscale diff failed at 17 31 [h264 @ 0x559cf8501580] error while decoding MB 17 31, bytestream 4604 [h2", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:42:53+09:00", "last_error_summary": "[h264 @ 0x559cf84ac540] error while decoding MB 14 25, bytestream -8 [h264 @ 0x559cf90603c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559cf90603c0] error while decoding MB 0 34, bytestream 3453 [h264 @ 0x559cf84c0700] top block unavailable for requested intra mode [h264 @ 0x559cf84c0700] error while decoding MB 43 0, bytestream 5302 [h264 @ 0x559cf8501580] cabac decode of qscale diff failed at 17 31 [h264 @ 0x559cf8501580] error while decoding MB 17 31, bytestream 4604 [h2", "last_exit_at": "2026-07-06T15:31:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 696.982, "last_started_at": "2026-07-06T15:31:16+09:00", "pid": 13777, "process": {"exists": false, "pid": 13777}, "restart_count": 22, "restart_count_per_hour": 7, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.326, "sample_count": 2788, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95041, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:42:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T15:42:53+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T15:42:53+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T15:42:53.118934+09:00", "mtime_epoch": 1783320173.1189342, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208006}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:42:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1485.582, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.709, "pushed": 3.709, "read": 3.709}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.372, "count": 111, "max_ms": 5.097, "p95_ms": 4.777}, "gstreamer_push": {"avg_ms": 1.471, "count": 111, "max_ms": 7.913, "p95_ms": 3.417}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.277, "count": 111, "max_ms": 80.495, "p95_ms": 69.839}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.843, "count": 111, "max_ms": 112.675, "p95_ms": 103.079}, "yolo11_640_fire_11class": {"avg_ms": 59.756, "count": 111, "max_ms": 94.518, "p95_ms": 80.451}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.65, "count": 111, "max_ms": 95.048, "p95_ms": 89.572}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.864, "count": 111, "max_ms": 125.79, "p95_ms": 115.939}, "read_interval": {"avg_ms": 270.229, "count": 111, "max_ms": 10120.709, "p95_ms": 131.954}, "triton_wait": {"avg_ms": 71.916, "count": 111, "max_ms": 112.689, "p95_ms": 103.092}, "visualize": {"avg_ms": 3.59, "count": 111, "max_ms": 4.542, "p95_ms": 4.356}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 792}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5539.938, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:42:53+09:00", "last_error_summary": "[h264 @ 0x559cf84ac540] error while decoding MB 14 25, bytestream -8 [h264 @ 0x559cf90603c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559cf90603c0] error while decoding MB 0 34, bytestream 3453 [h264 @ 0x559cf84c0700] top block unavailable for requested intra mode [h264 @ 0x559cf84c0700] error while decoding MB 43 0, bytestream 5302 [h264 @ 0x559cf8501580] cabac decode of qscale diff failed at 17 31 [h264 @ 0x559cf8501580] error while decoding MB 17 31, bytestream 4604 [h2", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:42:53+09:00", "last_error_summary": "[h264 @ 0x559cf84ac540] error while decoding MB 14 25, bytestream -8 [h264 @ 0x559cf90603c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559cf90603c0] error while decoding MB 0 34, bytestream 3453 [h264 @ 0x559cf84c0700] top block unavailable for requested intra mode [h264 @ 0x559cf84c0700] error while decoding MB 43 0, bytestream 5302 [h264 @ 0x559cf8501580] cabac decode of qscale diff failed at 17 31 [h264 @ 0x559cf8501580] error while decoding MB 17 31, bytestream 4604 [h2", "last_exit_at": "2026-07-06T15:42:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 697.041, "last_started_at": "2026-07-06T15:31:16+09:00", "pid": 13777, "process": {"exists": false, "pid": 13777}, "restart_count": 22, "restart_count_per_hour": 7, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.326, "sample_count": 2788, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95042, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:42:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T15:42:53+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T15:42:53+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-06T15:42:53.118934+09:00", "mtime_epoch": 1783320173.1189342, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208006}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:42:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1485.582, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.735, "pushed": 3.735, "read": 3.702}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.36, "count": 112, "max_ms": 5.097, "p95_ms": 4.777}, "gstreamer_push": {"avg_ms": 1.468, "count": 112, "max_ms": 7.913, "p95_ms": 3.417}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.467, "count": 112, "max_ms": 80.495, "p95_ms": 71.433}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.994, "count": 112, "max_ms": 112.675, "p95_ms": 103.079}, "yolo11_640_fire_11class": {"avg_ms": 59.885, "count": 112, "max_ms": 94.518, "p95_ms": 80.451}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.854, "count": 112, "max_ms": 95.048, "p95_ms": 89.572}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.987, "count": 112, "max_ms": 125.79, "p95_ms": 115.939}, "read_interval": {"avg_ms": 270.229, "count": 111, "max_ms": 10120.709, "p95_ms": 131.954}, "triton_wait": {"avg_ms": 72.066, "count": 112, "max_ms": 112.689, "p95_ms": 103.092}, "visualize": {"avg_ms": 3.581, "count": 112, "max_ms": 4.542, "p95_ms": 4.356}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 792}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5542.68, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:42:53+09:00", "last_error_summary": "[h264 @ 0x559cf84ac540] error while decoding MB 14 25, bytestream -8 [h264 @ 0x559cf90603c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559cf90603c0] error while decoding MB 0 34, bytestream 3453 [h264 @ 0x559cf84c0700] top block unavailable for requested intra mode [h264 @ 0x559cf84c0700] error while decoding MB 43 0, bytestream 5302 [h264 @ 0x559cf8501580] cabac decode of qscale diff failed at 17 31 [h264 @ 0x559cf8501580] error while decoding MB 17 31, bytestream 4604 [h2", "last_exit_at": "2026-07-06T15:42:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 697.048, "last_started_at": "2026-07-06T15:31:16+09:00", "pid": 13777, "process": {"exists": false, "pid": 13777}, "restart_count": 22, "restart_count_per_hour": 7, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.326, "sample_count": 2788, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95042, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:42:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T15:42:53+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T15:42:53+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T15:42:53.118934+09:00", "mtime_epoch": 1783320173.1189342, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208006}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:42:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1485.582, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.735, "pushed": 3.735, "read": 3.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.36, "count": 112, "max_ms": 5.097, "p95_ms": 4.777}, "gstreamer_push": {"avg_ms": 1.468, "count": 112, "max_ms": 7.913, "p95_ms": 3.417}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.467, "count": 112, "max_ms": 80.495, "p95_ms": 71.433}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.994, "count": 112, "max_ms": 112.675, "p95_ms": 103.079}, "yolo11_640_fire_11class": {"avg_ms": 59.885, "count": 112, "max_ms": 94.518, "p95_ms": 80.451}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.854, "count": 112, "max_ms": 95.048, "p95_ms": 89.572}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.987, "count": 112, "max_ms": 125.79, "p95_ms": 115.939}, "read_interval": {"avg_ms": 270.229, "count": 111, "max_ms": 10120.709, "p95_ms": 131.954}, "triton_wait": {"avg_ms": 72.066, "count": 112, "max_ms": 112.689, "p95_ms": 103.092}, "visualize": {"avg_ms": 3.581, "count": 112, "max_ms": 4.542, "p95_ms": 4.356}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 792}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5561.637, "limit_mb": 8796093022207.996}, "detail": {"pid": 14188, "restart_count": 23, "start_count": 24}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 63, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:42:53+09:00", "last_error_summary": "[h264 @ 0x559cf84ac540] error while decoding MB 14 25, bytestream -8 [h264 @ 0x559cf90603c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559cf90603c0] error while decoding MB 0 34, bytestream 3453 [h264 @ 0x559cf84c0700] top block unavailable for requested intra mode [h264 @ 0x559cf84c0700] error while decoding MB 43 0, bytestream 5302 [h264 @ 0x559cf8501580] cabac decode of qscale diff failed at 17 31 [h264 @ 0x559cf8501580] error while decoding MB 17 31, bytestream 4604 [h2", "last_exit_at": "2026-07-06T15:42:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:42:56+09:00", "pid": 14188, "process": {"exists": true, "fd_count": 4, "hwm_mb": 52.996, "name": "ffmpeg", "pid": 14188, "rss_mb": 52.996, "state": "S (sleeping)", "threads": 1}, "restart_count": 23, "restart_count_per_hour": 8, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.33, "sample_count": 2740, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95042, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:42:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.816, "last_frame_pushed_at": "2026-07-06T15:42:53+09:00", "last_frame_read_age_sec": 4.915, "last_frame_read_at": "2026-07-06T15:42:53+09:00", "mjpeg": {"age_sec": 4.799, "exists": true, "mtime": "2026-07-06T15:42:53.226937+09:00", "mtime_epoch": 1783320173.2269368, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207955}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:42:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1485.582, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 61, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.033, "pushed": 2.033, "read": 2.033}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.488, "count": 61, "max_ms": 5.097, "p95_ms": 4.873}, "gstreamer_push": {"avg_ms": 1.358, "count": 61, "max_ms": 3.545, "p95_ms": 1.478}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.54, "count": 61, "max_ms": 76.989, "p95_ms": 72.56}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.548, "count": 61, "max_ms": 107.826, "p95_ms": 97.0}, "yolo11_640_fire_11class": {"avg_ms": 59.657, "count": 61, "max_ms": 85.632, "p95_ms": 80.262}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.765, "count": 61, "max_ms": 92.135, "p95_ms": 84.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.897, "count": 61, "max_ms": 121.99, "p95_ms": 109.306}, "read_interval": {"avg_ms": 413.073, "count": 61, "max_ms": 10120.709, "p95_ms": 131.003}, "triton_wait": {"avg_ms": 72.604, "count": 61, "max_ms": 107.842, "p95_ms": 97.011}, "visualize": {"avg_ms": 3.722, "count": 61, "max_ms": 4.542, "p95_ms": 4.431}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 823}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5537.129, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5579d8bf1380] reference picture missing during reorder [h264 @ 0x5579d8bf1380] Missing reference picture, default is 65542 [h264 @ 0x5579d8c05540] mmco: unref short failure [h264 @ 0x5579d8c05540] illegal short term buffer state detected [h264 @ 0x5579d8bdd1c0] error while decoding MB 6 36, bytestream -10 [h264 @ 0x5579d8bc9000] left block unavailable for requested intra mode [h264 @ 0x5579d8bc9000] error while decoding MB 0 27, bytestream 57364 [h264 @ 0x5579d8b48f80] left block unava", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:49:54+09:00", "last_error_summary": "[h264 @ 0x5579d8bf1380] reference picture missing during reorder [h264 @ 0x5579d8bf1380] Missing reference picture, default is 65542 [h264 @ 0x5579d8c05540] mmco: unref short failure [h264 @ 0x5579d8c05540] illegal short term buffer state detected [h264 @ 0x5579d8bdd1c0] error while decoding MB 6 36, bytestream -10 [h264 @ 0x5579d8bc9000] left block unavailable for requested intra mode [h264 @ 0x5579d8bc9000] error while decoding MB 0 27, bytestream 57364 [h264 @ 0x5579d8b48f80] left block unava", "last_exit_at": "2026-07-06T15:42:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 418.247, "last_started_at": "2026-07-06T15:42:56+09:00", "pid": 14188, "process": {"exists": false, "pid": 14188}, "restart_count": 23, "restart_count_per_hour": 7, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.858, "sample_count": 2618, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 98821, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:49:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T15:49:54+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T15:49:54+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T15:49:54.215912+09:00", "mtime_epoch": 1783320594.215912, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194492}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:49:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1484.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 106, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.54, "pushed": 3.54, "read": 3.54}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.329, "count": 106, "max_ms": 4.757, "p95_ms": 3.998}, "gstreamer_push": {"avg_ms": 1.453, "count": 106, "max_ms": 3.935, "p95_ms": 3.611}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.997, "count": 106, "max_ms": 87.042, "p95_ms": 73.177}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.853, "count": 106, "max_ms": 116.376, "p95_ms": 103.974}, "yolo11_640_fire_11class": {"avg_ms": 60.598, "count": 106, "max_ms": 89.95, "p95_ms": 83.708}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.529, "count": 106, "max_ms": 95.143, "p95_ms": 91.275}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.968, "count": 106, "max_ms": 128.78, "p95_ms": 115.861}, "read_interval": {"avg_ms": 282.532, "count": 106, "max_ms": 10196.073, "p95_ms": 134.573}, "triton_wait": {"avg_ms": 72.894, "count": 106, "max_ms": 116.389, "p95_ms": 103.986}, "visualize": {"avg_ms": 3.682, "count": 106, "max_ms": 4.636, "p95_ms": 4.029}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 823}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5531.195, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:49:54+09:00", "last_error_summary": "[h264 @ 0x5579d8bf1380] reference picture missing during reorder [h264 @ 0x5579d8bf1380] Missing reference picture, default is 65542 [h264 @ 0x5579d8c05540] mmco: unref short failure [h264 @ 0x5579d8c05540] illegal short term buffer state detected [h264 @ 0x5579d8bdd1c0] error while decoding MB 6 36, bytestream -10 [h264 @ 0x5579d8bc9000] left block unavailable for requested intra mode [h264 @ 0x5579d8bc9000] error while decoding MB 0 27, bytestream 57364 [h264 @ 0x5579d8b48f80] left block unava", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:49:54+09:00", "last_error_summary": "[h264 @ 0x5579d8bf1380] reference picture missing during reorder [h264 @ 0x5579d8bf1380] Missing reference picture, default is 65542 [h264 @ 0x5579d8c05540] mmco: unref short failure [h264 @ 0x5579d8c05540] illegal short term buffer state detected [h264 @ 0x5579d8bdd1c0] error while decoding MB 6 36, bytestream -10 [h264 @ 0x5579d8bc9000] left block unavailable for requested intra mode [h264 @ 0x5579d8bc9000] error while decoding MB 0 27, bytestream 57364 [h264 @ 0x5579d8b48f80] left block unava", "last_exit_at": "2026-07-06T15:49:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 418.299, "last_started_at": "2026-07-06T15:42:56+09:00", "pid": 14188, "process": {"exists": false, "pid": 14188}, "restart_count": 23, "restart_count_per_hour": 7, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.858, "sample_count": 2619, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 98822, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:49:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T15:49:54+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-06T15:49:54+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T15:49:54.215912+09:00", "mtime_epoch": 1783320594.215912, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194492}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:49:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1479.012, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.567, "pushed": 3.567, "read": 3.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.328, "count": 107, "max_ms": 4.757, "p95_ms": 3.998}, "gstreamer_push": {"avg_ms": 1.472, "count": 107, "max_ms": 3.935, "p95_ms": 3.611}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.097, "count": 107, "max_ms": 87.042, "p95_ms": 73.177}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.058, "count": 107, "max_ms": 116.376, "p95_ms": 103.974}, "yolo11_640_fire_11class": {"avg_ms": 60.819, "count": 107, "max_ms": 89.95, "p95_ms": 84.235}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.732, "count": 107, "max_ms": 95.143, "p95_ms": 91.275}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.18, "count": 107, "max_ms": 128.78, "p95_ms": 115.861}, "read_interval": {"avg_ms": 282.532, "count": 106, "max_ms": 10196.073, "p95_ms": 134.573}, "triton_wait": {"avg_ms": 73.099, "count": 107, "max_ms": 116.389, "p95_ms": 103.986}, "visualize": {"avg_ms": 3.681, "count": 107, "max_ms": 4.636, "p95_ms": 4.029}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 823}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5531.262, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:49:54+09:00", "last_error_summary": "[h264 @ 0x5579d8bf1380] reference picture missing during reorder [h264 @ 0x5579d8bf1380] Missing reference picture, default is 65542 [h264 @ 0x5579d8c05540] mmco: unref short failure [h264 @ 0x5579d8c05540] illegal short term buffer state detected [h264 @ 0x5579d8bdd1c0] error while decoding MB 6 36, bytestream -10 [h264 @ 0x5579d8bc9000] left block unavailable for requested intra mode [h264 @ 0x5579d8bc9000] error while decoding MB 0 27, bytestream 57364 [h264 @ 0x5579d8b48f80] left block unava", "last_exit_at": "2026-07-06T15:49:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 418.307, "last_started_at": "2026-07-06T15:42:56+09:00", "pid": 14188, "process": {"exists": false, "pid": 14188}, "restart_count": 23, "restart_count_per_hour": 7, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.858, "sample_count": 2619, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 98822, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:49:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T15:49:54+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T15:49:54+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T15:49:54.215912+09:00", "mtime_epoch": 1783320594.215912, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194492}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:49:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1479.012, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 106, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.535, "pushed": 3.535, "read": 3.535}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.329, "count": 106, "max_ms": 4.757, "p95_ms": 3.998}, "gstreamer_push": {"avg_ms": 1.472, "count": 106, "max_ms": 3.935, "p95_ms": 3.611}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.137, "count": 106, "max_ms": 87.042, "p95_ms": 73.177}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.192, "count": 106, "max_ms": 116.376, "p95_ms": 103.974}, "yolo11_640_fire_11class": {"avg_ms": 60.904, "count": 106, "max_ms": 89.95, "p95_ms": 84.235}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.815, "count": 106, "max_ms": 95.143, "p95_ms": 91.275}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.31, "count": 106, "max_ms": 128.78, "p95_ms": 115.861}, "read_interval": {"avg_ms": 282.532, "count": 106, "max_ms": 10196.073, "p95_ms": 134.573}, "triton_wait": {"avg_ms": 73.233, "count": 106, "max_ms": 116.389, "p95_ms": 103.986}, "visualize": {"avg_ms": 3.678, "count": 106, "max_ms": 4.636, "p95_ms": 4.029}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 823}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5641.305, "limit_mb": 8796093022207.996}, "detail": {"pid": 14295, "restart_count": 24, "start_count": 25}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:49:54+09:00", "last_error_summary": "[h264 @ 0x5579d8bf1380] reference picture missing during reorder [h264 @ 0x5579d8bf1380] Missing reference picture, default is 65542 [h264 @ 0x5579d8c05540] mmco: unref short failure [h264 @ 0x5579d8c05540] illegal short term buffer state detected [h264 @ 0x5579d8bdd1c0] error while decoding MB 6 36, bytestream -10 [h264 @ 0x5579d8bc9000] left block unavailable for requested intra mode [h264 @ 0x5579d8bc9000] error while decoding MB 0 27, bytestream 57364 [h264 @ 0x5579d8b48f80] left block unava", "last_exit_at": "2026-07-06T15:49:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:49:56+09:00", "pid": 14295, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.875, "name": "ffmpeg", "pid": 14295, "rss_mb": 136.875, "state": "S (sleeping)", "threads": 106}, "restart_count": 24, "restart_count_per_hour": 8, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 126.188, "sample_count": 2577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 98822, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:49:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.081, "last_frame_pushed_at": "2026-07-06T15:49:54+09:00", "last_frame_read_age_sec": 4.193, "last_frame_read_at": "2026-07-06T15:49:54+09:00", "mjpeg": {"age_sec": 4.064, "exists": true, "mtime": "2026-07-06T15:49:54.335914+09:00", "mtime_epoch": 1783320594.3359144, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193779}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T15:49:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1479.012, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.135, "pushed": 2.135, "read": 2.101}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.325, "count": 64, "max_ms": 4.117, "p95_ms": 3.935}, "gstreamer_push": {"avg_ms": 1.572, "count": 64, "max_ms": 3.935, "p95_ms": 3.804}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.418, "count": 64, "max_ms": 83.752, "p95_ms": 73.111}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.271, "count": 64, "max_ms": 116.376, "p95_ms": 107.816}, "yolo11_640_fire_11class": {"avg_ms": 61.751, "count": 64, "max_ms": 89.95, "p95_ms": 85.352}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.399, "count": 64, "max_ms": 95.143, "p95_ms": 93.43}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.41, "count": 64, "max_ms": 128.78, "p95_ms": 119.844}, "read_interval": {"avg_ms": 410.373, "count": 63, "max_ms": 10196.073, "p95_ms": 139.589}, "triton_wait": {"avg_ms": 75.314, "count": 64, "max_ms": 116.389, "p95_ms": 107.828}, "visualize": {"avg_ms": 3.682, "count": 64, "max_ms": 4.636, "p95_ms": 4.023}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 711}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5479.012, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5577fb1d2e00] error while decoding MB 50 61, bytestream -27", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:52:52+09:00", "last_error_summary": "[h264 @ 0x5577fb1d2e00] error while decoding MB 50 61, bytestream -27", "last_exit_at": "2026-07-06T15:36:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 993.651, "last_started_at": "2026-07-06T15:36:19+09:00", "pid": 14036, "process": {"exists": false, "pid": 14036}, "restart_count": 32, "restart_count_per_hour": 9, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.845, "sample_count": 2280, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85436, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:52:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T15:52:52+09:00", "last_frame_read_age_sec": 0.073, "last_frame_read_at": "2026-07-06T15:52:52+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-06T15:52:52.887375+09:00", "mtime_epoch": 1783320772.8873749, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217716}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:52:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1346.875, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.596, "pushed": 7.596, "read": 7.562}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.467, "count": 227, "max_ms": 10.129, "p95_ms": 8.834}, "gstreamer_push": {"avg_ms": 1.257, "count": 227, "max_ms": 4.518, "p95_ms": 1.689}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.638, "count": 227, "max_ms": 83.383, "p95_ms": 75.613}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.963, "count": 227, "max_ms": 116.197, "p95_ms": 98.268}, "yolo11_640_fire_11class": {"avg_ms": 58.953, "count": 227, "max_ms": 83.586, "p95_ms": 78.316}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.885, "count": 227, "max_ms": 100.048, "p95_ms": 92.241}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.961, "count": 227, "max_ms": 168.487, "p95_ms": 151.289}, "read_interval": {"avg_ms": 132.517, "count": 226, "max_ms": 178.309, "p95_ms": 161.687}, "triton_wait": {"avg_ms": 71.68, "count": 227, "max_ms": 116.21, "p95_ms": 98.282}, "visualize": {"avg_ms": 5.155, "count": 227, "max_ms": 8.998, "p95_ms": 6.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 711}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5478.91, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:52:52+09:00", "last_error_summary": "[h264 @ 0x5577fb1d2e00] error while decoding MB 50 61, bytestream -27", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:52:52+09:00", "last_error_summary": "[h264 @ 0x5577fb1d2e00] error while decoding MB 50 61, bytestream -27", "last_exit_at": "2026-07-06T15:52:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 993.667, "last_started_at": "2026-07-06T15:36:19+09:00", "pid": 14036, "process": {"exists": false, "pid": 14036}, "restart_count": 32, "restart_count_per_hour": 9, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.845, "sample_count": 2280, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85436, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:52:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.022, "last_frame_pushed_at": "2026-07-06T15:52:52+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-06T15:52:52+09:00", "mjpeg": {"age_sec": 0.079, "exists": true, "mtime": "2026-07-06T15:52:52.887375+09:00", "mtime_epoch": 1783320772.8873749, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217716}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:52:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1346.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.593, "pushed": 7.593, "read": 7.559}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.467, "count": 227, "max_ms": 10.129, "p95_ms": 8.834}, "gstreamer_push": {"avg_ms": 1.257, "count": 227, "max_ms": 4.518, "p95_ms": 1.689}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.638, "count": 227, "max_ms": 83.383, "p95_ms": 75.613}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.963, "count": 227, "max_ms": 116.197, "p95_ms": 98.268}, "yolo11_640_fire_11class": {"avg_ms": 58.953, "count": 227, "max_ms": 83.586, "p95_ms": 78.316}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.885, "count": 227, "max_ms": 100.048, "p95_ms": 92.241}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.961, "count": 227, "max_ms": 168.487, "p95_ms": 151.289}, "read_interval": {"avg_ms": 132.517, "count": 226, "max_ms": 178.309, "p95_ms": 161.687}, "triton_wait": {"avg_ms": 71.68, "count": 227, "max_ms": 116.21, "p95_ms": 98.282}, "visualize": {"avg_ms": 5.155, "count": 227, "max_ms": 8.998, "p95_ms": 6.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 711}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5478.684, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:52:52+09:00", "last_error_summary": "[h264 @ 0x5577fb1d2e00] error while decoding MB 50 61, bytestream -27", "last_exit_at": "2026-07-06T15:52:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 993.677, "last_started_at": "2026-07-06T15:36:19+09:00", "pid": 14036, "process": {"exists": false, "pid": 14036}, "restart_count": 32, "restart_count_per_hour": 9, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.845, "sample_count": 2280, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85436, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:52:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.032, "last_frame_pushed_at": "2026-07-06T15:52:52+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-06T15:52:52+09:00", "mjpeg": {"age_sec": 0.012, "exists": true, "mtime": "2026-07-06T15:52:52.963376+09:00", "mtime_epoch": 1783320772.9633763, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233613}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:52:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1346.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.59, "pushed": 7.59, "read": 7.557}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.467, "count": 227, "max_ms": 10.129, "p95_ms": 8.834}, "gstreamer_push": {"avg_ms": 1.257, "count": 227, "max_ms": 4.518, "p95_ms": 1.689}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.638, "count": 227, "max_ms": 83.383, "p95_ms": 75.613}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.963, "count": 227, "max_ms": 116.197, "p95_ms": 98.268}, "yolo11_640_fire_11class": {"avg_ms": 58.953, "count": 227, "max_ms": 83.586, "p95_ms": 78.316}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.885, "count": 227, "max_ms": 100.048, "p95_ms": 92.241}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.961, "count": 227, "max_ms": 168.487, "p95_ms": 151.289}, "read_interval": {"avg_ms": 132.517, "count": 226, "max_ms": 178.309, "p95_ms": 161.687}, "triton_wait": {"avg_ms": 71.68, "count": 227, "max_ms": 116.21, "p95_ms": 98.282}, "visualize": {"avg_ms": 5.155, "count": 227, "max_ms": 8.998, "p95_ms": 6.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 711}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5593.129, "limit_mb": 8796093022207.996}, "detail": {"pid": 14402, "restart_count": 33, "start_count": 34}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:52:52+09:00", "last_error_summary": "[h264 @ 0x5577fb1d2e00] error while decoding MB 50 61, bytestream -27", "last_exit_at": "2026-07-06T15:52:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:52:55+09:00", "pid": 14402, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.824, "name": "ffmpeg", "pid": 14402, "rss_mb": 156.824, "state": "R (running)", "threads": 49}, "restart_count": 33, "restart_count_per_hour": 10, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.939, "sample_count": 2245, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85436, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:52:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.216, "last_frame_pushed_at": "2026-07-06T15:52:52+09:00", "last_frame_read_age_sec": 4.283, "last_frame_read_at": "2026-07-06T15:52:52+09:00", "mjpeg": {"age_sec": 4.196, "exists": true, "mtime": "2026-07-06T15:52:52.963376+09:00", "mtime_epoch": 1783320772.9633763, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233613}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:52:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1346.875, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.555, "pushed": 6.555, "read": 6.522}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.461, "count": 196, "max_ms": 10.129, "p95_ms": 9.206}, "gstreamer_push": {"avg_ms": 1.278, "count": 196, "max_ms": 4.518, "p95_ms": 2.194}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.639, "count": 196, "max_ms": 80.189, "p95_ms": 75.549}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.589, "count": 196, "max_ms": 109.344, "p95_ms": 97.835}, "yolo11_640_fire_11class": {"avg_ms": 58.75, "count": 196, "max_ms": 83.586, "p95_ms": 77.512}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.854, "count": 196, "max_ms": 96.788, "p95_ms": 92.241}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.391, "count": 196, "max_ms": 161.33, "p95_ms": 149.836}, "read_interval": {"avg_ms": 132.03, "count": 195, "max_ms": 173.736, "p95_ms": 160.749}, "triton_wait": {"avg_ms": 71.35, "count": 196, "max_ms": 109.355, "p95_ms": 97.848}, "visualize": {"avg_ms": 5.142, "count": 196, "max_ms": 8.998, "p95_ms": 6.31}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 719}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5473.691, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d2cfaba280] error while decoding MB 57 31, bytestream -9 [in#0/rtsp @ 0x55d2ceda7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d2ceda7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:55:12+09:00", "last_error_summary": "[h264 @ 0x55d2cfaba280] error while decoding MB 57 31, bytestream -9 [in#0/rtsp @ 0x55d2ceda7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d2ceda7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:52:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.467, "last_started_at": "2026-07-06T15:52:55+09:00", "pid": 14402, "process": {"exists": false, "pid": 14402}, "restart_count": 33, "restart_count_per_hour": 9, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.274, "sample_count": 2056, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 86293, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:55:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T15:55:12+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-06T15:55:12+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-06T15:55:12.573995+09:00", "mtime_epoch": 1783320912.573995, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244572}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:55:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1347.074, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 48, "pushed_frames": 48, "read_frames": 48, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.604, "pushed": 1.604, "read": 1.604}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.82, "count": 48, "max_ms": 9.354, "p95_ms": 6.633}, "gstreamer_push": {"avg_ms": 1.261, "count": 48, "max_ms": 3.925, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.129, "count": 48, "max_ms": 75.432, "p95_ms": 71.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.781, "count": 48, "max_ms": 111.588, "p95_ms": 93.961}, "yolo11_640_fire_11class": {"avg_ms": 62.498, "count": 48, "max_ms": 85.289, "p95_ms": 84.342}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.849, "count": 48, "max_ms": 100.242, "p95_ms": 86.009}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.036, "count": 48, "max_ms": 165.982, "p95_ms": 146.277}, "read_interval": {"avg_ms": 624.555, "count": 48, "max_ms": 15367.875, "p95_ms": 174.759}, "triton_wait": {"avg_ms": 70.86, "count": 48, "max_ms": 111.601, "p95_ms": 93.974}, "visualize": {"avg_ms": 4.755, "count": 48, "max_ms": 6.144, "p95_ms": 5.479}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 719}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5473.43, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T15:55:12+09:00", "last_error_summary": "[h264 @ 0x55d2cfaba280] error while decoding MB 57 31, bytestream -9 [in#0/rtsp @ 0x55d2ceda7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d2ceda7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:55:12+09:00", "last_error_summary": "[h264 @ 0x55d2cfaba280] error while decoding MB 57 31, bytestream -9 [in#0/rtsp @ 0x55d2ceda7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d2ceda7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:55:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.487, "last_started_at": "2026-07-06T15:52:55+09:00", "pid": 14402, "process": {"exists": false, "pid": 14402}, "restart_count": 33, "restart_count_per_hour": 9, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.274, "sample_count": 2057, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 86294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:55:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T15:55:12+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-06T15:55:12+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-06T15:55:12.573995+09:00", "mtime_epoch": 1783320912.573995, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244572}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:55:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1347.074, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.636, "pushed": 1.636, "read": 1.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.802, "count": 49, "max_ms": 9.354, "p95_ms": 6.633}, "gstreamer_push": {"avg_ms": 1.262, "count": 49, "max_ms": 3.925, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.87, "count": 49, "max_ms": 75.432, "p95_ms": 71.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.372, "count": 49, "max_ms": 111.588, "p95_ms": 93.961}, "yolo11_640_fire_11class": {"avg_ms": 62.225, "count": 49, "max_ms": 85.289, "p95_ms": 84.342}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.651, "count": 49, "max_ms": 100.242, "p95_ms": 86.009}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.2, "count": 49, "max_ms": 165.982, "p95_ms": 146.277}, "read_interval": {"avg_ms": 624.555, "count": 48, "max_ms": 15367.875, "p95_ms": 174.759}, "triton_wait": {"avg_ms": 70.54, "count": 49, "max_ms": 111.601, "p95_ms": 93.974}, "visualize": {"avg_ms": 4.796, "count": 49, "max_ms": 6.796, "p95_ms": 5.69}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 719}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5473.422, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:55:12+09:00", "last_error_summary": "[h264 @ 0x55d2cfaba280] error while decoding MB 57 31, bytestream -9 [in#0/rtsp @ 0x55d2ceda7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d2ceda7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:55:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.494, "last_started_at": "2026-07-06T15:52:55+09:00", "pid": 14402, "process": {"exists": false, "pid": 14402}, "restart_count": 33, "restart_count_per_hour": 9, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.274, "sample_count": 2057, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 86294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:55:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T15:55:12+09:00", "last_frame_read_age_sec": 0.087, "last_frame_read_at": "2026-07-06T15:55:12+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-06T15:55:12.573995+09:00", "mtime_epoch": 1783320912.573995, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244572}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:55:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1347.074, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.636, "pushed": 1.636, "read": 1.602}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.795, "count": 48, "max_ms": 9.354, "p95_ms": 6.633}, "gstreamer_push": {"avg_ms": 1.262, "count": 49, "max_ms": 3.925, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.077, "count": 48, "max_ms": 75.432, "p95_ms": 71.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.716, "count": 48, "max_ms": 111.588, "p95_ms": 93.961}, "yolo11_640_fire_11class": {"avg_ms": 62.498, "count": 48, "max_ms": 85.289, "p95_ms": 84.342}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.887, "count": 48, "max_ms": 100.242, "p95_ms": 86.009}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.2, "count": 49, "max_ms": 165.982, "p95_ms": 146.277}, "read_interval": {"avg_ms": 624.555, "count": 48, "max_ms": 15367.875, "p95_ms": 174.759}, "triton_wait": {"avg_ms": 70.888, "count": 48, "max_ms": 111.601, "p95_ms": 93.974}, "visualize": {"avg_ms": 4.793, "count": 48, "max_ms": 6.796, "p95_ms": 5.69}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 719}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5600.602, "limit_mb": 8796093022207.996}, "detail": {"pid": 14554, "restart_count": 34, "start_count": 35}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T15:55:12+09:00", "last_error_summary": "[h264 @ 0x55d2cfaba280] error while decoding MB 57 31, bytestream -9 [in#0/rtsp @ 0x55d2ceda7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d2ceda7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:55:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T15:55:14+09:00", "pid": 14554, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.457, "name": "ffmpeg", "pid": 14554, "rss_mb": 156.457, "state": "R (running)", "threads": 49}, "restart_count": 34, "restart_count_per_hour": 10, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.514, "sample_count": 2025, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 86294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T15:55:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.244, "last_frame_pushed_at": "2026-07-06T15:55:12+09:00", "last_frame_read_age_sec": 4.318, "last_frame_read_at": "2026-07-06T15:55:12+09:00", "mjpeg": {"age_sec": 4.225, "exists": true, "mtime": "2026-07-06T15:55:12.657997+09:00", "mtime_epoch": 1783320912.6579967, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238508}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T15:55:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1347.074, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.467, "pushed": 0.467, "read": 0.434}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.764, "count": 14, "max_ms": 9.343, "p95_ms": 9.343}, "gstreamer_push": {"avg_ms": 1.154, "count": 14, "max_ms": 1.368, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.634, "count": 14, "max_ms": 73.294, "p95_ms": 73.294}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.842, "count": 14, "max_ms": 111.588, "p95_ms": 111.588}, "yolo11_640_fire_11class": {"avg_ms": 66.463, "count": 14, "max_ms": 84.207, "p95_ms": 84.207}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.245, "count": 14, "max_ms": 84.55, "p95_ms": 84.55}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.227, "count": 14, "max_ms": 165.982, "p95_ms": 165.982}, "read_interval": {"avg_ms": 1980.043, "count": 13, "max_ms": 15367.875, "p95_ms": 15367.875}, "triton_wait": {"avg_ms": 76.266, "count": 14, "max_ms": 111.601, "p95_ms": 111.601}, "visualize": {"avg_ms": 4.9, "count": 14, "max_ms": 6.796, "p95_ms": 6.796}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 870}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5543.469, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:00:11+09:00", "last_error_summary": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "last_exit_at": "2026-07-06T15:49:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 615.395, "last_started_at": "2026-07-06T15:49:56+09:00", "pid": 14295, "process": {"exists": false, "pid": 14295}, "restart_count": 24, "restart_count_per_hour": 7, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.058, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104462, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:00:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T16:00:11+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-06T16:00:11+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T16:00:11.743434+09:00", "mtime_epoch": 1783321211.7434344, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:00:11+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1489.512, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.6, "pushed": 6.6, "read": 6.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.517, "count": 197, "max_ms": 10.71, "p95_ms": 7.962}, "gstreamer_push": {"avg_ms": 1.17, "count": 197, "max_ms": 4.46, "p95_ms": 1.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.883, "count": 197, "max_ms": 79.357, "p95_ms": 71.372}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.634, "count": 197, "max_ms": 118.503, "p95_ms": 103.99}, "yolo11_640_fire_11class": {"avg_ms": 60.686, "count": 197, "max_ms": 91.328, "p95_ms": 81.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.12, "count": 197, "max_ms": 100.465, "p95_ms": 94.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.637, "count": 197, "max_ms": 199.826, "p95_ms": 181.509}, "read_interval": {"avg_ms": 152.075, "count": 197, "max_ms": 207.646, "p95_ms": 190.134}, "triton_wait": {"avg_ms": 76.803, "count": 197, "max_ms": 118.516, "p95_ms": 104.004}, "visualize": {"avg_ms": 5.789, "count": 197, "max_ms": 8.302, "p95_ms": 7.004}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 870}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5548.629, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:00:11+09:00", "last_error_summary": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:00:11+09:00", "last_error_summary": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "last_exit_at": "2026-07-06T16:00:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 615.464, "last_started_at": "2026-07-06T15:49:56+09:00", "pid": 14295, "process": {"exists": false, "pid": 14295}, "restart_count": 24, "restart_count_per_hour": 7, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.058, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104463, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:00:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T16:00:11+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T16:00:11+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T16:00:11.743434+09:00", "mtime_epoch": 1783321211.7434344, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:00:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1493.121, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.619, "pushed": 6.619, "read": 6.585}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.534, "count": 198, "max_ms": 10.71, "p95_ms": 8.081}, "gstreamer_push": {"avg_ms": 1.17, "count": 198, "max_ms": 4.46, "p95_ms": 1.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.915, "count": 198, "max_ms": 79.357, "p95_ms": 71.372}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.582, "count": 198, "max_ms": 118.503, "p95_ms": 103.99}, "yolo11_640_fire_11class": {"avg_ms": 60.674, "count": 198, "max_ms": 91.328, "p95_ms": 81.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.101, "count": 198, "max_ms": 100.465, "p95_ms": 94.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.533, "count": 198, "max_ms": 199.826, "p95_ms": 181.509}, "read_interval": {"avg_ms": 152.075, "count": 197, "max_ms": 207.646, "p95_ms": 190.134}, "triton_wait": {"avg_ms": 76.75, "count": 198, "max_ms": 118.516, "p95_ms": 104.004}, "visualize": {"avg_ms": 5.784, "count": 198, "max_ms": 8.302, "p95_ms": 7.004}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 870}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5548.59, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:00:11+09:00", "last_error_summary": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "last_exit_at": "2026-07-06T16:00:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 615.469, "last_started_at": "2026-07-06T15:49:56+09:00", "pid": 14295, "process": {"exists": false, "pid": 14295}, "restart_count": 24, "restart_count_per_hour": 7, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.058, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104463, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:00:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T16:00:11+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T16:00:11+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-06T16:00:11.743434+09:00", "mtime_epoch": 1783321211.7434344, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:00:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1493.121, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.617, "pushed": 6.617, "read": 6.584}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.534, "count": 198, "max_ms": 10.71, "p95_ms": 8.081}, "gstreamer_push": {"avg_ms": 1.17, "count": 198, "max_ms": 4.46, "p95_ms": 1.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.915, "count": 198, "max_ms": 79.357, "p95_ms": 71.372}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.582, "count": 198, "max_ms": 118.503, "p95_ms": 103.99}, "yolo11_640_fire_11class": {"avg_ms": 60.674, "count": 198, "max_ms": 91.328, "p95_ms": 81.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.101, "count": 198, "max_ms": 100.465, "p95_ms": 94.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.533, "count": 198, "max_ms": 199.826, "p95_ms": 181.509}, "read_interval": {"avg_ms": 152.075, "count": 197, "max_ms": 207.646, "p95_ms": 190.134}, "triton_wait": {"avg_ms": 76.75, "count": 198, "max_ms": 118.516, "p95_ms": 104.004}, "visualize": {"avg_ms": 5.784, "count": 198, "max_ms": 8.302, "p95_ms": 7.004}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 870}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5659.992, "limit_mb": 8796093022207.996}, "detail": {"pid": 14706, "restart_count": 25, "start_count": 26}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:00:11+09:00", "last_error_summary": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "last_exit_at": "2026-07-06T16:00:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:00:14+09:00", "pid": 14706, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.52, "name": "ffmpeg", "pid": 14706, "rss_mb": 136.52, "state": "S (sleeping)", "threads": 106}, "restart_count": 25, "restart_count_per_hour": 8, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.39, "sample_count": 2509, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104463, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:00:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.807, "last_frame_pushed_at": "2026-07-06T16:00:11+09:00", "last_frame_read_age_sec": 4.931, "last_frame_read_at": "2026-07-06T16:00:11+09:00", "mjpeg": {"age_sec": 4.79, "exists": true, "mtime": "2026-07-06T16:00:11.875437+09:00", "mtime_epoch": 1783321211.8754368, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194721}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:00:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1692.938, "name": "celery", "pid": 237, "rss_mb": 1493.121, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 170, "pushed_frames": 170, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.681, "pushed": 5.681, "read": 5.647}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.47, "count": 170, "max_ms": 10.71, "p95_ms": 7.962}, "gstreamer_push": {"avg_ms": 1.189, "count": 170, "max_ms": 4.46, "p95_ms": 1.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.591, "count": 170, "max_ms": 79.357, "p95_ms": 69.534}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.556, "count": 170, "max_ms": 113.602, "p95_ms": 102.684}, "yolo11_640_fire_11class": {"avg_ms": 60.689, "count": 170, "max_ms": 91.328, "p95_ms": 81.349}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.87, "count": 170, "max_ms": 100.465, "p95_ms": 93.988}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.964, "count": 170, "max_ms": 189.075, "p95_ms": 174.867}, "read_interval": {"avg_ms": 148.619, "count": 169, "max_ms": 198.416, "p95_ms": 185.661}, "triton_wait": {"avg_ms": 75.749, "count": 170, "max_ms": 113.615, "p95_ms": 102.697}, "visualize": {"avg_ms": 5.672, "count": 170, "max_ms": 8.302, "p95_ms": 6.691}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 744}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5479.332, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f0fdc069c0] error while decoding MB 71 63, bytestream -23 [in#0/rtsp @ 0x55f0fd09ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0fd09ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:01:32+09:00", "last_error_summary": "[h264 @ 0x55f0fdc069c0] error while decoding MB 71 63, bytestream -23 [in#0/rtsp @ 0x55f0fd09ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0fd09ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T15:55:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 377.263, "last_started_at": "2026-07-06T15:55:14+09:00", "pid": 14554, "process": {"exists": false, "pid": 14554}, "restart_count": 34, "restart_count_per_hour": 8, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.388, "sample_count": 2508, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 89382, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:01:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T16:01:32+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T16:01:32+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-06T16:01:32.100867+09:00", "mtime_epoch": 1783321292.1008668, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 246320}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:01:32+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1349.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 222, "pushed_frames": 222, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.511, "pushed": 7.511, "read": 7.545}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.333, "count": 222, "max_ms": 10.174, "p95_ms": 7.71}, "gstreamer_push": {"avg_ms": 1.181, "count": 222, "max_ms": 4.508, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.836, "count": 222, "max_ms": 82.767, "p95_ms": 67.499}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.854, "count": 222, "max_ms": 114.916, "p95_ms": 99.552}, "yolo11_640_fire_11class": {"avg_ms": 61.963, "count": 222, "max_ms": 88.495, "p95_ms": 77.582}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.938, "count": 222, "max_ms": 97.849, "p95_ms": 89.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.251, "count": 222, "max_ms": 173.94, "p95_ms": 154.332}, "read_interval": {"avg_ms": 186.385, "count": 223, "max_ms": 12063.423, "p95_ms": 163.732}, "triton_wait": {"avg_ms": 70.394, "count": 222, "max_ms": 114.929, "p95_ms": 99.566}, "visualize": {"avg_ms": 5.075, "count": 222, "max_ms": 7.891, "p95_ms": 5.828}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.556, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 744}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5482.344, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:01:32+09:00", "last_error_summary": "[h264 @ 0x55f0fdc069c0] error while decoding MB 71 63, bytestream -23 [in#0/rtsp @ 0x55f0fd09ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0fd09ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:01:32+09:00", "last_error_summary": "[h264 @ 0x55f0fdc069c0] error while decoding MB 71 63, bytestream -23 [in#0/rtsp @ 0x55f0fd09ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0fd09ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:01:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 377.331, "last_started_at": "2026-07-06T15:55:14+09:00", "pid": 14554, "process": {"exists": false, "pid": 14554}, "restart_count": 34, "restart_count_per_hour": 8, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.388, "sample_count": 2509, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 89383, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:01:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T16:01:32+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-06T16:01:32+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T16:01:32.100867+09:00", "mtime_epoch": 1783321292.1008668, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 246320}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:01:32+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1349.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.528, "pushed": 7.528, "read": 7.528}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.341, "count": 223, "max_ms": 10.174, "p95_ms": 7.782}, "gstreamer_push": {"avg_ms": 1.18, "count": 223, "max_ms": 4.508, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.853, "count": 223, "max_ms": 82.767, "p95_ms": 67.499}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.811, "count": 223, "max_ms": 114.916, "p95_ms": 99.552}, "yolo11_640_fire_11class": {"avg_ms": 61.95, "count": 223, "max_ms": 88.495, "p95_ms": 77.582}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.911, "count": 223, "max_ms": 97.849, "p95_ms": 89.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.217, "count": 223, "max_ms": 173.94, "p95_ms": 154.332}, "read_interval": {"avg_ms": 186.385, "count": 223, "max_ms": 12063.423, "p95_ms": 163.732}, "triton_wait": {"avg_ms": 70.348, "count": 223, "max_ms": 114.929, "p95_ms": 99.566}, "visualize": {"avg_ms": 5.076, "count": 223, "max_ms": 7.891, "p95_ms": 5.828}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.624, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 744}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5476.219, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:01:32+09:00", "last_error_summary": "[h264 @ 0x55f0fdc069c0] error while decoding MB 71 63, bytestream -23 [in#0/rtsp @ 0x55f0fd09ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0fd09ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:01:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 377.337, "last_started_at": "2026-07-06T15:55:14+09:00", "pid": 14554, "process": {"exists": false, "pid": 14554}, "restart_count": 34, "restart_count_per_hour": 8, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.388, "sample_count": 2509, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 89383, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:01:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T16:01:32+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T16:01:32+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-06T16:01:32.100867+09:00", "mtime_epoch": 1783321292.1008668, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 246320}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:01:32+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1349.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.526, "pushed": 7.526, "read": 7.526}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.341, "count": 223, "max_ms": 10.174, "p95_ms": 7.782}, "gstreamer_push": {"avg_ms": 1.18, "count": 223, "max_ms": 4.508, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.853, "count": 223, "max_ms": 82.767, "p95_ms": 67.499}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.811, "count": 223, "max_ms": 114.916, "p95_ms": 99.552}, "yolo11_640_fire_11class": {"avg_ms": 61.95, "count": 223, "max_ms": 88.495, "p95_ms": 77.582}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.911, "count": 223, "max_ms": 97.849, "p95_ms": 89.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.217, "count": 223, "max_ms": 173.94, "p95_ms": 154.332}, "read_interval": {"avg_ms": 186.385, "count": 223, "max_ms": 12063.423, "p95_ms": 163.732}, "triton_wait": {"avg_ms": 70.348, "count": 223, "max_ms": 114.929, "p95_ms": 99.566}, "visualize": {"avg_ms": 5.076, "count": 223, "max_ms": 7.891, "p95_ms": 5.828}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.63, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 744}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6031.109, "limit_mb": 8796093022207.996}, "detail": {"pid": 14874, "restart_count": 35, "start_count": 36}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:01:32+09:00", "last_error_summary": "[h264 @ 0x55f0fdc069c0] error while decoding MB 71 63, bytestream -23 [in#0/rtsp @ 0x55f0fd09ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0fd09ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:01:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:01:35+09:00", "pid": 14874, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.262, "name": "ffmpeg", "pid": 14874, "rss_mb": 206.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 35, "restart_count_per_hour": 9, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.373, "sample_count": 2471, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 89383, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:01:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.88, "last_frame_pushed_at": "2026-07-06T16:01:32+09:00", "last_frame_read_age_sec": 4.997, "last_frame_read_at": "2026-07-06T16:01:32+09:00", "mjpeg": {"age_sec": 4.863, "exists": true, "mtime": "2026-07-06T16:01:32.224869+09:00", "mtime_epoch": 1783321292.224869, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 245692}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:01:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1349.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.37, "pushed": 6.37, "read": 6.337}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.345, "count": 190, "max_ms": 10.174, "p95_ms": 7.973}, "gstreamer_push": {"avg_ms": 1.181, "count": 191, "max_ms": 4.508, "p95_ms": 2.348}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.708, "count": 190, "max_ms": 79.047, "p95_ms": 67.499}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.074, "count": 190, "max_ms": 104.914, "p95_ms": 96.179}, "yolo11_640_fire_11class": {"avg_ms": 61.719, "count": 190, "max_ms": 88.495, "p95_ms": 77.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.851, "count": 190, "max_ms": 97.849, "p95_ms": 89.238}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.665, "count": 191, "max_ms": 173.94, "p95_ms": 152.885}, "read_interval": {"avg_ms": 132.333, "count": 190, "max_ms": 184.897, "p95_ms": 162.081}, "triton_wait": {"avg_ms": 69.644, "count": 190, "max_ms": 104.928, "p95_ms": 96.201}, "visualize": {"avg_ms": 5.056, "count": 190, "max_ms": 6.716, "p95_ms": 5.736}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 878}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5767.902, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:00:11+09:00", "last_error_summary": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:00:11+09:00", "last_error_summary": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "last_exit_at": "2026-07-06T16:02:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 147.311, "last_started_at": "2026-07-06T16:00:14+09:00", "pid": 14706, "process": {"exists": false, "pid": 14706}, "restart_count": 25, "restart_count_per_hour": 8, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.403, "sample_count": 2039, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105366, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:02:41+09:00", "last_frame_read_age_sec": 0.225, "last_frame_read_at": "2026-07-06T16:02:41+09:00", "mjpeg": {"age_sec": 0.213, "exists": true, "mtime": "2026-07-06T16:02:41.762101+09:00", "mtime_epoch": 1783321361.7621014, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 218895}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:02:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1694.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.608, "pushed": 6.608, "read": 6.574}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.57, "count": 197, "max_ms": 11.892, "p95_ms": 9.357}, "gstreamer_push": {"avg_ms": 1.081, "count": 198, "max_ms": 1.503, "p95_ms": 1.239}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.32, "count": 197, "max_ms": 88.397, "p95_ms": 78.559}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.49, "count": 197, "max_ms": 118.873, "p95_ms": 107.489}, "yolo11_640_fire_11class": {"avg_ms": 65.424, "count": 197, "max_ms": 91.338, "p95_ms": 86.714}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.939, "count": 197, "max_ms": 106.127, "p95_ms": 97.321}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.468, "count": 198, "max_ms": 250.04, "p95_ms": 207.339}, "read_interval": {"avg_ms": 151.548, "count": 197, "max_ms": 257.525, "p95_ms": 211.824}, "triton_wait": {"avg_ms": 81.709, "count": 197, "max_ms": 118.886, "p95_ms": 107.501}, "visualize": {"avg_ms": 6.008, "count": 197, "max_ms": 17.918, "p95_ms": 12.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 878}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5767.895, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:00:11+09:00", "last_error_summary": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "last_exit_at": "2026-07-06T16:02:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 147.315, "last_started_at": "2026-07-06T16:00:14+09:00", "pid": 14706, "process": {"exists": false, "pid": 14706}, "restart_count": 25, "restart_count_per_hour": 8, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.403, "sample_count": 2039, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105366, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T16:02:41+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-06T16:02:41+09:00", "mjpeg": {"age_sec": 0.217, "exists": true, "mtime": "2026-07-06T16:02:41.762101+09:00", "mtime_epoch": 1783321361.7621014, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 218895}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:02:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1694.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.607, "pushed": 6.607, "read": 6.573}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.57, "count": 197, "max_ms": 11.892, "p95_ms": 9.357}, "gstreamer_push": {"avg_ms": 1.081, "count": 198, "max_ms": 1.503, "p95_ms": 1.239}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.32, "count": 197, "max_ms": 88.397, "p95_ms": 78.559}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.49, "count": 197, "max_ms": 118.873, "p95_ms": 107.489}, "yolo11_640_fire_11class": {"avg_ms": 65.424, "count": 197, "max_ms": 91.338, "p95_ms": 86.714}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.939, "count": 197, "max_ms": 106.127, "p95_ms": 97.321}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.468, "count": 198, "max_ms": 250.04, "p95_ms": 207.339}, "read_interval": {"avg_ms": 151.548, "count": 197, "max_ms": 257.525, "p95_ms": 211.824}, "triton_wait": {"avg_ms": 81.709, "count": 197, "max_ms": 118.886, "p95_ms": 107.501}, "visualize": {"avg_ms": 6.008, "count": 197, "max_ms": 17.918, "p95_ms": 12.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 878}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5872.902, "limit_mb": 8796093022207.996}, "detail": {"pid": 15026, "restart_count": 26, "start_count": 27}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:00:11+09:00", "last_error_summary": "[h264 @ 0x55fbb380d100] cabac decode of qscale diff failed at 55 28 [h264 @ 0x55fbb380d100] error while decoding MB 55 28, bytestream -7", "last_exit_at": "2026-07-06T16:02:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:02:44+09:00", "pid": 15026, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.598, "name": "ffmpeg", "pid": 15026, "rss_mb": 136.598, "state": "S (sleeping)", "threads": 106}, "restart_count": 26, "restart_count_per_hour": 9, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.571, "sample_count": 2003, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105366, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.122, "last_frame_pushed_at": "2026-07-06T16:02:41+09:00", "last_frame_read_age_sec": 4.343, "last_frame_read_at": "2026-07-06T16:02:41+09:00", "mjpeg": {"age_sec": 4.099, "exists": true, "mtime": "2026-07-06T16:02:41.994106+09:00", "mtime_epoch": 1783321361.9941056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 219209}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:02:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1694.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 169, "pushed_frames": 170, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.633, "pushed": 5.667, "read": 5.633}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.617, "count": 169, "max_ms": 11.892, "p95_ms": 9.523}, "gstreamer_push": {"avg_ms": 1.081, "count": 170, "max_ms": 1.503, "p95_ms": 1.241}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.91, "count": 169, "max_ms": 88.397, "p95_ms": 78.559}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.529, "count": 169, "max_ms": 117.798, "p95_ms": 107.489}, "yolo11_640_fire_11class": {"avg_ms": 65.456, "count": 169, "max_ms": 91.338, "p95_ms": 86.714}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.068, "count": 169, "max_ms": 106.127, "p95_ms": 97.396}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.703, "count": 169, "max_ms": 250.04, "p95_ms": 207.749}, "read_interval": {"avg_ms": 152.67, "count": 169, "max_ms": 257.525, "p95_ms": 213.954}, "triton_wait": {"avg_ms": 81.674, "count": 169, "max_ms": 117.811, "p95_ms": 107.501}, "visualize": {"avg_ms": 6.122, "count": 169, "max_ms": 17.918, "p95_ms": 12.308}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 751}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5638.469, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5558995c9600] error while decoding MB 77 35, bytestream -25 [in#0/rtsp @ 0x555899434e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555899434e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:03:40+09:00", "last_error_summary": "[h264 @ 0x5558995c9600] error while decoding MB 77 35, bytestream -25 [in#0/rtsp @ 0x555899434e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555899434e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:01:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.004, "last_started_at": "2026-07-06T16:01:35+09:00", "pid": 14874, "process": {"exists": false, "pid": 14874}, "restart_count": 35, "restart_count_per_hour": 9, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.549, "sample_count": 2244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 90171, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:03:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-06T16:03:40+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T16:03:40+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T16:03:40.051130+09:00", "mtime_epoch": 1783321420.05113, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230270}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:03:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1349.449, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 43, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.438, "pushed": 1.438, "read": 1.438}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.9, "count": 43, "max_ms": 9.115, "p95_ms": 6.796}, "gstreamer_push": {"avg_ms": 1.227, "count": 43, "max_ms": 3.196, "p95_ms": 1.578}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.859, "count": 43, "max_ms": 79.145, "p95_ms": 76.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.277, "count": 43, "max_ms": 114.458, "p95_ms": 102.842}, "yolo11_640_fire_11class": {"avg_ms": 65.083, "count": 43, "max_ms": 88.004, "p95_ms": 85.939}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.949, "count": 43, "max_ms": 101.366, "p95_ms": 97.136}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.309, "count": 43, "max_ms": 173.574, "p95_ms": 162.624}, "read_interval": {"avg_ms": 697.301, "count": 43, "max_ms": 15975.707, "p95_ms": 188.678}, "triton_wait": {"avg_ms": 76.488, "count": 43, "max_ms": 114.47, "p95_ms": 102.855}, "visualize": {"avg_ms": 4.829, "count": 43, "max_ms": 6.184, "p95_ms": 5.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 751}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5637.773, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:03:40+09:00", "last_error_summary": "[h264 @ 0x5558995c9600] error while decoding MB 77 35, bytestream -25 [in#0/rtsp @ 0x555899434e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555899434e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:03:40+09:00", "last_error_summary": "[h264 @ 0x5558995c9600] error while decoding MB 77 35, bytestream -25 [in#0/rtsp @ 0x555899434e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555899434e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:03:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.098, "last_started_at": "2026-07-06T16:01:35+09:00", "pid": 14874, "process": {"exists": false, "pid": 14874}, "restart_count": 35, "restart_count_per_hour": 9, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.549, "sample_count": 2244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 90172, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:03:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:03:40+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-06T16:03:40+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-06T16:03:40.051130+09:00", "mtime_epoch": 1783321420.05113, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230270}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:03:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1349.449, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.467, "pushed": 1.467, "read": 1.433}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.906, "count": 43, "max_ms": 9.115, "p95_ms": 6.796}, "gstreamer_push": {"avg_ms": 1.225, "count": 44, "max_ms": 3.196, "p95_ms": 1.578}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.165, "count": 43, "max_ms": 79.145, "p95_ms": 76.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.373, "count": 43, "max_ms": 114.458, "p95_ms": 102.842}, "yolo11_640_fire_11class": {"avg_ms": 65.171, "count": 43, "max_ms": 88.004, "p95_ms": 85.939}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.013, "count": 43, "max_ms": 101.366, "p95_ms": 97.136}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.692, "count": 44, "max_ms": 173.574, "p95_ms": 162.624}, "read_interval": {"avg_ms": 697.301, "count": 43, "max_ms": 15975.707, "p95_ms": 188.678}, "triton_wait": {"avg_ms": 76.585, "count": 43, "max_ms": 114.47, "p95_ms": 102.855}, "visualize": {"avg_ms": 4.836, "count": 43, "max_ms": 6.184, "p95_ms": 5.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 751}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5637.891, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:03:40+09:00", "last_error_summary": "[h264 @ 0x5558995c9600] error while decoding MB 77 35, bytestream -25 [in#0/rtsp @ 0x555899434e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555899434e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:03:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.103, "last_started_at": "2026-07-06T16:01:35+09:00", "pid": 14874, "process": {"exists": false, "pid": 14874}, "restart_count": 35, "restart_count_per_hour": 9, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.549, "sample_count": 2244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 90172, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:03:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T16:03:40+09:00", "last_frame_read_age_sec": 0.153, "last_frame_read_at": "2026-07-06T16:03:40+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-06T16:03:40.051130+09:00", "mtime_epoch": 1783321420.05113, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230270}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:03:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1349.449, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.433, "pushed": 1.467, "read": 1.433}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.906, "count": 43, "max_ms": 9.115, "p95_ms": 6.796}, "gstreamer_push": {"avg_ms": 1.225, "count": 44, "max_ms": 3.196, "p95_ms": 1.578}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.165, "count": 43, "max_ms": 79.145, "p95_ms": 76.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.373, "count": 43, "max_ms": 114.458, "p95_ms": 102.842}, "yolo11_640_fire_11class": {"avg_ms": 65.171, "count": 43, "max_ms": 88.004, "p95_ms": 85.939}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.013, "count": 43, "max_ms": 101.366, "p95_ms": 97.136}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.456, "count": 43, "max_ms": 173.574, "p95_ms": 162.624}, "read_interval": {"avg_ms": 697.301, "count": 43, "max_ms": 15975.707, "p95_ms": 188.678}, "triton_wait": {"avg_ms": 76.585, "count": 43, "max_ms": 114.47, "p95_ms": 102.855}, "visualize": {"avg_ms": 4.836, "count": 43, "max_ms": 6.184, "p95_ms": 5.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 751}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5795.555, "limit_mb": 8796093022207.996}, "detail": {"pid": 15133, "restart_count": 36, "start_count": 37}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:03:40+09:00", "last_error_summary": "[h264 @ 0x5558995c9600] error while decoding MB 77 35, bytestream -25 [in#0/rtsp @ 0x555899434e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555899434e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:03:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:03:42+09:00", "pid": 15133, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.945, "name": "ffmpeg", "pid": 15133, "rss_mb": 194.945, "state": "S (sleeping)", "threads": 106}, "restart_count": 36, "restart_count_per_hour": 10, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.649, "sample_count": 2206, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 90172, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:03:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.056, "last_frame_pushed_at": "2026-07-06T16:03:40+09:00", "last_frame_read_age_sec": 4.199, "last_frame_read_at": "2026-07-06T16:03:40+09:00", "mjpeg": {"age_sec": 4.037, "exists": true, "mtime": "2026-07-06T16:03:40.199133+09:00", "mtime_epoch": 1783321420.1991327, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:03:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1349.449, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 15, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.5, "pushed": 0.5, "read": 0.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.336, "count": 14, "max_ms": 6.796, "p95_ms": 6.796}, "gstreamer_push": {"avg_ms": 1.26, "count": 15, "max_ms": 2.59, "p95_ms": 2.59}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.422, "count": 14, "max_ms": 79.145, "p95_ms": 79.145}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.01, "count": 14, "max_ms": 102.071, "p95_ms": 102.071}, "yolo11_640_fire_11class": {"avg_ms": 62.531, "count": 14, "max_ms": 85.939, "p95_ms": 85.939}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.714, "count": 14, "max_ms": 101.366, "p95_ms": 101.366}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.96, "count": 15, "max_ms": 172.44, "p95_ms": 172.44}, "read_interval": {"avg_ms": 1853.53, "count": 14, "max_ms": 15975.707, "p95_ms": 15975.707}, "triton_wait": {"avg_ms": 74.237, "count": 14, "max_ms": 102.084, "p95_ms": 102.084}, "visualize": {"avg_ms": 4.463, "count": 14, "max_ms": 5.467, "p95_ms": 5.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 884}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5576.023, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56102f148d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56102f148d00] error while decoding MB 0 6, bytestream 96083 [h264 @ 0x56102f120980] left block unavailable for requested intra mode [h264 @ 0x56102f120980] error while decoding MB 0 17, bytestream 1196 [h264 @ 0x56102fdc4b00] left block unavailable for requested intra mode [h264 @ 0x56102fdc4b00] error while decoding MB 0 28, bytestream 4822 [h264 @ 0x56102f15cec0] left block unavailable for requested intra4x", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 169, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:05:03+09:00", "last_error_summary": "[h264 @ 0x56102f148d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56102f148d00] error while decoding MB 0 6, bytestream 96083 [h264 @ 0x56102f120980] left block unavailable for requested intra mode [h264 @ 0x56102f120980] error while decoding MB 0 17, bytestream 1196 [h264 @ 0x56102fdc4b00] left block unavailable for requested intra mode [h264 @ 0x56102fdc4b00] error while decoding MB 0 28, bytestream 4822 [h264 @ 0x56102f15cec0] left block unavailable for requested intra4x", "last_exit_at": "2026-07-06T16:02:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.213, "last_started_at": "2026-07-06T16:02:44+09:00", "pid": 15026, "process": {"exists": false, "pid": 15026}, "restart_count": 26, "restart_count_per_hour": 8, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.814, "sample_count": 1789, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 106190, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:05:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T16:05:03+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T16:05:03+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-06T16:05:03.256592+09:00", "mtime_epoch": 1783321503.256592, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:05:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1515.008, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 78, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.603, "pushed": 2.603, "read": 2.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.717, "count": 77, "max_ms": 9.314, "p95_ms": 9.001}, "gstreamer_push": {"avg_ms": 1.145, "count": 78, "max_ms": 1.898, "p95_ms": 1.426}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.948, "count": 77, "max_ms": 79.492, "p95_ms": 76.697}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.578, "count": 77, "max_ms": 113.442, "p95_ms": 109.053}, "yolo11_640_fire_11class": {"avg_ms": 65.243, "count": 77, "max_ms": 88.864, "p95_ms": 80.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.195, "count": 77, "max_ms": 101.654, "p95_ms": 97.328}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.152, "count": 78, "max_ms": 177.716, "p95_ms": 164.388}, "read_interval": {"avg_ms": 385.246, "count": 78, "max_ms": 10209.398, "p95_ms": 184.733}, "triton_wait": {"avg_ms": 83.077, "count": 77, "max_ms": 113.454, "p95_ms": 109.066}, "visualize": {"avg_ms": 4.938, "count": 77, "max_ms": 8.55, "p95_ms": 6.142}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 884}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5577.594, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:05:03+09:00", "last_error_summary": "[h264 @ 0x56102f148d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56102f148d00] error while decoding MB 0 6, bytestream 96083 [h264 @ 0x56102f120980] left block unavailable for requested intra mode [h264 @ 0x56102f120980] error while decoding MB 0 17, bytestream 1196 [h264 @ 0x56102fdc4b00] left block unavailable for requested intra mode [h264 @ 0x56102fdc4b00] error while decoding MB 0 28, bytestream 4822 [h264 @ 0x56102f15cec0] left block unavailable for requested intra4x", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 169, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:05:03+09:00", "last_error_summary": "[h264 @ 0x56102f148d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56102f148d00] error while decoding MB 0 6, bytestream 96083 [h264 @ 0x56102f120980] left block unavailable for requested intra mode [h264 @ 0x56102f120980] error while decoding MB 0 17, bytestream 1196 [h264 @ 0x56102fdc4b00] left block unavailable for requested intra mode [h264 @ 0x56102fdc4b00] error while decoding MB 0 28, bytestream 4822 [h264 @ 0x56102f15cec0] left block unavailable for requested intra4x", "last_exit_at": "2026-07-06T16:05:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.296, "last_started_at": "2026-07-06T16:02:44+09:00", "pid": 15026, "process": {"exists": false, "pid": 15026}, "restart_count": 26, "restart_count_per_hour": 8, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.814, "sample_count": 1790, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 106191, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:05:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:05:03+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-06T16:05:03+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-06T16:05:03.256592+09:00", "mtime_epoch": 1783321503.256592, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:05:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1515.008, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 77, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.61, "pushed": 2.61, "read": 2.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.719, "count": 78, "max_ms": 9.314, "p95_ms": 9.001}, "gstreamer_push": {"avg_ms": 1.142, "count": 78, "max_ms": 1.898, "p95_ms": 1.426}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.868, "count": 78, "max_ms": 79.492, "p95_ms": 76.697}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.68, "count": 78, "max_ms": 113.442, "p95_ms": 109.053}, "yolo11_640_fire_11class": {"avg_ms": 65.339, "count": 78, "max_ms": 88.864, "p95_ms": 80.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.35, "count": 78, "max_ms": 101.654, "p95_ms": 97.328}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.048, "count": 78, "max_ms": 177.716, "p95_ms": 164.388}, "read_interval": {"avg_ms": 388.164, "count": 77, "max_ms": 10209.398, "p95_ms": 184.733}, "triton_wait": {"avg_ms": 83.173, "count": 78, "max_ms": 113.454, "p95_ms": 109.066}, "visualize": {"avg_ms": 4.941, "count": 78, "max_ms": 8.55, "p95_ms": 6.142}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 884}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5577.48, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 169, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:05:03+09:00", "last_error_summary": "[h264 @ 0x56102f148d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56102f148d00] error while decoding MB 0 6, bytestream 96083 [h264 @ 0x56102f120980] left block unavailable for requested intra mode [h264 @ 0x56102f120980] error while decoding MB 0 17, bytestream 1196 [h264 @ 0x56102fdc4b00] left block unavailable for requested intra mode [h264 @ 0x56102fdc4b00] error while decoding MB 0 28, bytestream 4822 [h264 @ 0x56102f15cec0] left block unavailable for requested intra4x", "last_exit_at": "2026-07-06T16:05:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.301, "last_started_at": "2026-07-06T16:02:44+09:00", "pid": 15026, "process": {"exists": false, "pid": 15026}, "restart_count": 26, "restart_count_per_hour": 8, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.814, "sample_count": 1790, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 106191, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:05:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T16:05:03+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-06T16:05:03+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-06T16:05:03.256592+09:00", "mtime_epoch": 1783321503.256592, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:05:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1515.008, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 77, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.61, "pushed": 2.61, "read": 2.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.719, "count": 78, "max_ms": 9.314, "p95_ms": 9.001}, "gstreamer_push": {"avg_ms": 1.142, "count": 78, "max_ms": 1.898, "p95_ms": 1.426}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.868, "count": 78, "max_ms": 79.492, "p95_ms": 76.697}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.68, "count": 78, "max_ms": 113.442, "p95_ms": 109.053}, "yolo11_640_fire_11class": {"avg_ms": 65.339, "count": 78, "max_ms": 88.864, "p95_ms": 80.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.35, "count": 78, "max_ms": 101.654, "p95_ms": 97.328}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.048, "count": 78, "max_ms": 177.716, "p95_ms": 164.388}, "read_interval": {"avg_ms": 388.164, "count": 77, "max_ms": 10209.398, "p95_ms": 184.733}, "triton_wait": {"avg_ms": 83.173, "count": 78, "max_ms": 113.454, "p95_ms": 109.066}, "visualize": {"avg_ms": 4.941, "count": 78, "max_ms": 8.55, "p95_ms": 6.142}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 884}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5691.699, "limit_mb": 8796093022207.996}, "detail": {"pid": 15285, "restart_count": 27, "start_count": 28}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 169, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:05:03+09:00", "last_error_summary": "[h264 @ 0x56102f148d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56102f148d00] error while decoding MB 0 6, bytestream 96083 [h264 @ 0x56102f120980] left block unavailable for requested intra mode [h264 @ 0x56102f120980] error while decoding MB 0 17, bytestream 1196 [h264 @ 0x56102fdc4b00] left block unavailable for requested intra mode [h264 @ 0x56102fdc4b00] error while decoding MB 0 28, bytestream 4822 [h264 @ 0x56102f15cec0] left block unavailable for requested intra4x", "last_exit_at": "2026-07-06T16:05:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:05:05+09:00", "pid": 15285, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.59, "name": "ffmpeg", "pid": 15285, "rss_mb": 136.59, "state": "S (sleeping)", "threads": 106}, "restart_count": 27, "restart_count_per_hour": 9, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.899, "sample_count": 1760, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 106191, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:05:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.079, "last_frame_pushed_at": "2026-07-06T16:05:03+09:00", "last_frame_read_age_sec": 4.224, "last_frame_read_at": "2026-07-06T16:05:03+09:00", "mjpeg": {"age_sec": 4.063, "exists": true, "mtime": "2026-07-06T16:05:03.400594+09:00", "mtime_epoch": 1783321503.4005945, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201485}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:05:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1515.008, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.701, "pushed": 1.701, "read": 1.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.39, "count": 50, "max_ms": 8.211, "p95_ms": 7.609}, "gstreamer_push": {"avg_ms": 1.13, "count": 51, "max_ms": 1.372, "p95_ms": 1.306}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.459, "count": 50, "max_ms": 79.492, "p95_ms": 76.135}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.363, "count": 50, "max_ms": 113.442, "p95_ms": 109.293}, "yolo11_640_fire_11class": {"avg_ms": 64.57, "count": 50, "max_ms": 88.864, "p95_ms": 83.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.638, "count": 50, "max_ms": 101.654, "p95_ms": 97.328}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.443, "count": 51, "max_ms": 177.716, "p95_ms": 163.834}, "read_interval": {"avg_ms": 518.207, "count": 50, "max_ms": 10209.398, "p95_ms": 186.971}, "triton_wait": {"avg_ms": 82.125, "count": 50, "max_ms": 113.454, "p95_ms": 109.305}, "visualize": {"avg_ms": 4.689, "count": 50, "max_ms": 6.142, "p95_ms": 5.637}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 894}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5564.84, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:07:48+09:00", "last_error_summary": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "last_exit_at": "2026-07-06T16:05:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.505, "last_started_at": "2026-07-06T16:05:05+09:00", "pid": 15285, "process": {"exists": false, "pid": 15285}, "restart_count": 27, "restart_count_per_hour": 9, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.688, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 107280, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:07:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T16:07:48+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T16:07:48+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T16:07:48.927937+09:00", "mtime_epoch": 1783321668.9279368, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201095}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:07:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1486.691, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.686, "pushed": 6.686, "read": 6.686}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.044, "count": 200, "max_ms": 8.899, "p95_ms": 7.607}, "gstreamer_push": {"avg_ms": 1.236, "count": 200, "max_ms": 7.065, "p95_ms": 2.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.272, "count": 200, "max_ms": 86.821, "p95_ms": 78.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.993, "count": 200, "max_ms": 126.311, "p95_ms": 105.154}, "yolo11_640_fire_11class": {"avg_ms": 66.695, "count": 200, "max_ms": 86.416, "p95_ms": 81.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.67, "count": 200, "max_ms": 105.709, "p95_ms": 100.003}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.237, "count": 200, "max_ms": 184.286, "p95_ms": 161.146}, "read_interval": {"avg_ms": 149.831, "count": 200, "max_ms": 192.396, "p95_ms": 171.914}, "triton_wait": {"avg_ms": 86.511, "count": 200, "max_ms": 126.323, "p95_ms": 105.722}, "visualize": {"avg_ms": 5.117, "count": 200, "max_ms": 6.315, "p95_ms": 6.006}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 894}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5564.082, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:07:48+09:00", "last_error_summary": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:07:48+09:00", "last_error_summary": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "last_exit_at": "2026-07-06T16:07:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.602, "last_started_at": "2026-07-06T16:05:05+09:00", "pid": 15285, "process": {"exists": false, "pid": 15285}, "restart_count": 27, "restart_count_per_hour": 9, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.688, "sample_count": 1895, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 107281, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:07:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T16:07:49+09:00", "last_frame_read_age_sec": 0.15, "last_frame_read_at": "2026-07-06T16:07:48+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-06T16:07:48.927937+09:00", "mtime_epoch": 1783321668.9279368, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201095}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:07:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1486.691, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.698, "pushed": 6.698, "read": 6.664}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.045, "count": 200, "max_ms": 8.899, "p95_ms": 7.607}, "gstreamer_push": {"avg_ms": 1.237, "count": 200, "max_ms": 7.065, "p95_ms": 2.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.437, "count": 200, "max_ms": 86.821, "p95_ms": 78.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.201, "count": 200, "max_ms": 126.311, "p95_ms": 105.154}, "yolo11_640_fire_11class": {"avg_ms": 66.801, "count": 200, "max_ms": 86.416, "p95_ms": 81.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.858, "count": 200, "max_ms": 105.709, "p95_ms": 100.003}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.435, "count": 200, "max_ms": 184.286, "p95_ms": 161.146}, "read_interval": {"avg_ms": 150.023, "count": 199, "max_ms": 192.396, "p95_ms": 172.462}, "triton_wait": {"avg_ms": 86.718, "count": 200, "max_ms": 126.323, "p95_ms": 105.722}, "visualize": {"avg_ms": 5.119, "count": 200, "max_ms": 6.315, "p95_ms": 6.006}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 894}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5564.137, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:07:48+09:00", "last_error_summary": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "last_exit_at": "2026-07-06T16:07:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.608, "last_started_at": "2026-07-06T16:05:05+09:00", "pid": 15285, "process": {"exists": false, "pid": 15285}, "restart_count": 27, "restart_count_per_hour": 9, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.688, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 107281, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:07:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T16:07:49+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-06T16:07:48+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T16:07:48.927937+09:00", "mtime_epoch": 1783321668.9279368, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201095}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:07:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1486.691, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.697, "pushed": 6.697, "read": 6.663}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.045, "count": 200, "max_ms": 8.899, "p95_ms": 7.607}, "gstreamer_push": {"avg_ms": 1.237, "count": 200, "max_ms": 7.065, "p95_ms": 2.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.437, "count": 200, "max_ms": 86.821, "p95_ms": 78.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.201, "count": 200, "max_ms": 126.311, "p95_ms": 105.154}, "yolo11_640_fire_11class": {"avg_ms": 66.801, "count": 200, "max_ms": 86.416, "p95_ms": 81.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.858, "count": 200, "max_ms": 105.709, "p95_ms": 100.003}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.435, "count": 200, "max_ms": 184.286, "p95_ms": 161.146}, "read_interval": {"avg_ms": 150.023, "count": 199, "max_ms": 192.396, "p95_ms": 172.462}, "triton_wait": {"avg_ms": 86.718, "count": 200, "max_ms": 126.323, "p95_ms": 105.722}, "visualize": {"avg_ms": 5.119, "count": 200, "max_ms": 6.315, "p95_ms": 6.006}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 894}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5671.754, "limit_mb": 8796093022207.996}, "detail": {"pid": 15392, "restart_count": 28, "start_count": 29}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:07:48+09:00", "last_error_summary": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "last_exit_at": "2026-07-06T16:07:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:07:51+09:00", "pid": 15392, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.082, "name": "ffmpeg", "pid": 15392, "rss_mb": 139.082, "state": "S (sleeping)", "threads": 106}, "restart_count": 28, "restart_count_per_hour": 10, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.689, "sample_count": 1862, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 107281, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:07:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.648, "last_frame_pushed_at": "2026-07-06T16:07:49+09:00", "last_frame_read_age_sec": 4.793, "last_frame_read_at": "2026-07-06T16:07:48+09:00", "mjpeg": {"age_sec": 4.628, "exists": true, "mtime": "2026-07-06T16:07:49.079941+09:00", "mtime_epoch": 1783321669.0799413, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201478}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:07:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1486.691, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 170, "pushed_frames": 170, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.669, "pushed": 5.669, "read": 5.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.051, "count": 169, "max_ms": 8.899, "p95_ms": 7.632}, "gstreamer_push": {"avg_ms": 1.235, "count": 170, "max_ms": 7.065, "p95_ms": 2.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.619, "count": 169, "max_ms": 86.821, "p95_ms": 78.334}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.283, "count": 169, "max_ms": 126.311, "p95_ms": 105.154}, "yolo11_640_fire_11class": {"avg_ms": 67.268, "count": 169, "max_ms": 86.416, "p95_ms": 81.645}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.094, "count": 169, "max_ms": 105.709, "p95_ms": 100.113}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.25, "count": 170, "max_ms": 184.286, "p95_ms": 162.392}, "read_interval": {"avg_ms": 149.759, "count": 169, "max_ms": 192.396, "p95_ms": 172.462}, "triton_wait": {"avg_ms": 86.779, "count": 169, "max_ms": 126.323, "p95_ms": 105.722}, "visualize": {"avg_ms": 5.129, "count": 169, "max_ms": 6.315, "p95_ms": 5.906}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 904}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5560.578, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:07:48+09:00", "last_error_summary": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:07:48+09:00", "last_error_summary": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "last_exit_at": "2026-07-06T16:10:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.455, "last_started_at": "2026-07-06T16:07:51+09:00", "pid": 15392, "process": {"exists": false, "pid": 15392}, "restart_count": 28, "restart_count_per_hour": 10, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.837, "sample_count": 2059, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108540, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:10:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T16:10:50+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-06T16:10:50+09:00", "mjpeg": {"age_sec": 0.152, "exists": true, "mtime": "2026-07-06T16:10:50.008993+09:00", "mtime_epoch": 1783321850.0089931, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204740}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:10:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1483.902, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.637, "pushed": 7.637, "read": 7.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.524, "count": 228, "max_ms": 9.33, "p95_ms": 6.909}, "gstreamer_push": {"avg_ms": 1.187, "count": 229, "max_ms": 3.976, "p95_ms": 1.391}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.184, "count": 228, "max_ms": 86.041, "p95_ms": 76.594}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.686, "count": 228, "max_ms": 118.238, "p95_ms": 99.079}, "yolo11_640_fire_11class": {"avg_ms": 60.463, "count": 228, "max_ms": 86.572, "p95_ms": 80.369}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.092, "count": 228, "max_ms": 102.137, "p95_ms": 93.278}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.955, "count": 229, "max_ms": 175.415, "p95_ms": 152.991}, "read_interval": {"avg_ms": 131.443, "count": 228, "max_ms": 188.133, "p95_ms": 162.087}, "triton_wait": {"avg_ms": 74.994, "count": 228, "max_ms": 118.255, "p95_ms": 100.101}, "visualize": {"avg_ms": 4.896, "count": 228, "max_ms": 8.583, "p95_ms": 5.546}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 904}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5560.57, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:07:48+09:00", "last_error_summary": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "last_exit_at": "2026-07-06T16:10:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.46, "last_started_at": "2026-07-06T16:07:51+09:00", "pid": 15392, "process": {"exists": false, "pid": 15392}, "restart_count": 28, "restart_count_per_hour": 10, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.837, "sample_count": 2059, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108540, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:10:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T16:10:50+09:00", "last_frame_read_age_sec": 0.166, "last_frame_read_at": "2026-07-06T16:10:50+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-06T16:10:50.008993+09:00", "mtime_epoch": 1783321850.0089931, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204740}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:10:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1483.902, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.635, "pushed": 7.635, "read": 7.602}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.524, "count": 228, "max_ms": 9.33, "p95_ms": 6.909}, "gstreamer_push": {"avg_ms": 1.187, "count": 229, "max_ms": 3.976, "p95_ms": 1.391}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.184, "count": 228, "max_ms": 86.041, "p95_ms": 76.594}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.686, "count": 228, "max_ms": 118.238, "p95_ms": 99.079}, "yolo11_640_fire_11class": {"avg_ms": 60.463, "count": 228, "max_ms": 86.572, "p95_ms": 80.369}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.092, "count": 228, "max_ms": 102.137, "p95_ms": 93.278}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.955, "count": 229, "max_ms": 175.415, "p95_ms": 152.991}, "read_interval": {"avg_ms": 131.443, "count": 228, "max_ms": 188.133, "p95_ms": 162.087}, "triton_wait": {"avg_ms": 74.994, "count": 228, "max_ms": 118.255, "p95_ms": 100.101}, "visualize": {"avg_ms": 4.896, "count": 228, "max_ms": 8.583, "p95_ms": 5.546}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 904}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5665.27, "limit_mb": 8796093022207.996}, "detail": {"pid": 15499, "restart_count": 29, "start_count": 30}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:07:48+09:00", "last_error_summary": "[h264 @ 0x560bcaf82ec0] error while decoding MB 11 27, bytestream -5", "last_exit_at": "2026-07-06T16:10:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:10:53+09:00", "pid": 15499, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.16, "name": "ffmpeg", "pid": 15499, "rss_mb": 136.16, "state": "S (sleeping)", "threads": 106}, "restart_count": 29, "restart_count_per_hour": 11, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.28, "sample_count": 2024, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108540, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:10:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.881, "last_frame_pushed_at": "2026-07-06T16:10:50+09:00", "last_frame_read_age_sec": 5.033, "last_frame_read_at": "2026-07-06T16:10:50+09:00", "mjpeg": {"age_sec": 4.865, "exists": true, "mtime": "2026-07-06T16:10:50.168998+09:00", "mtime_epoch": 1783321850.1689975, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205147}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:10:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1483.902, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.562, "pushed": 6.562, "read": 6.528}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.414, "count": 196, "max_ms": 9.33, "p95_ms": 6.405}, "gstreamer_push": {"avg_ms": 1.2, "count": 196, "max_ms": 3.976, "p95_ms": 1.47}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.996, "count": 196, "max_ms": 82.736, "p95_ms": 73.143}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.541, "count": 196, "max_ms": 118.238, "p95_ms": 96.739}, "yolo11_640_fire_11class": {"avg_ms": 58.762, "count": 196, "max_ms": 84.304, "p95_ms": 79.226}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.639, "count": 196, "max_ms": 101.295, "p95_ms": 92.095}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.685, "count": 196, "max_ms": 175.415, "p95_ms": 150.942}, "read_interval": {"avg_ms": 128.142, "count": 195, "max_ms": 188.133, "p95_ms": 160.043}, "triton_wait": {"avg_ms": 72.862, "count": 196, "max_ms": 118.255, "p95_ms": 97.879}, "visualize": {"avg_ms": 4.855, "count": 196, "max_ms": 8.583, "p95_ms": 5.546}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 777}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5458.102, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f0a15f0fc0] error while decoding MB 114 25, bytestream -37 [in#0/rtsp @ 0x55f0a1593e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0a1593e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:10:59+09:00", "last_error_summary": "[h264 @ 0x55f0a15f0fc0] error while decoding MB 114 25, bytestream -37 [in#0/rtsp @ 0x55f0a1593e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0a1593e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:03:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 437.53, "last_started_at": "2026-07-06T16:03:42+09:00", "pid": 15133, "process": {"exists": false, "pid": 15133}, "restart_count": 36, "restart_count_per_hour": 8, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.294, "sample_count": 2147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 93320, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:10:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T16:10:59+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T16:10:59+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T16:10:59.721250+09:00", "mtime_epoch": 1783321859.7212496, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:10:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1353.586, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 40, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.335, "pushed": 1.335, "read": 1.335}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.43, "count": 40, "max_ms": 8.69, "p95_ms": 7.528}, "gstreamer_push": {"avg_ms": 1.165, "count": 40, "max_ms": 2.793, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.97, "count": 40, "max_ms": 82.332, "p95_ms": 80.763}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.44, "count": 40, "max_ms": 115.38, "p95_ms": 101.947}, "yolo11_640_fire_11class": {"avg_ms": 65.676, "count": 40, "max_ms": 84.546, "p95_ms": 83.539}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.184, "count": 40, "max_ms": 99.764, "p95_ms": 95.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.471, "count": 40, "max_ms": 166.527, "p95_ms": 155.275}, "read_interval": {"avg_ms": 750.219, "count": 40, "max_ms": 15827.346, "p95_ms": 176.056}, "triton_wait": {"avg_ms": 77.845, "count": 40, "max_ms": 115.392, "p95_ms": 101.959}, "visualize": {"avg_ms": 4.949, "count": 40, "max_ms": 5.769, "p95_ms": 5.4}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 777}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5457.184, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:10:59+09:00", "last_error_summary": "[h264 @ 0x55f0a15f0fc0] error while decoding MB 114 25, bytestream -37 [in#0/rtsp @ 0x55f0a1593e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0a1593e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:10:59+09:00", "last_error_summary": "[h264 @ 0x55f0a15f0fc0] error while decoding MB 114 25, bytestream -37 [in#0/rtsp @ 0x55f0a1593e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0a1593e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:10:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 437.63, "last_started_at": "2026-07-06T16:03:42+09:00", "pid": 15133, "process": {"exists": false, "pid": 15133}, "restart_count": 36, "restart_count_per_hour": 8, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.294, "sample_count": 2147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 93321, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:10:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:10:59+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-06T16:10:59+09:00", "mjpeg": {"age_sec": 0.144, "exists": true, "mtime": "2026-07-06T16:10:59.721250+09:00", "mtime_epoch": 1783321859.7212496, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:10:59+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1353.586, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.338, "pushed": 1.338, "read": 1.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.453, "count": 40, "max_ms": 8.69, "p95_ms": 7.528}, "gstreamer_push": {"avg_ms": 1.167, "count": 40, "max_ms": 2.793, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.938, "count": 40, "max_ms": 82.332, "p95_ms": 80.763}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.523, "count": 40, "max_ms": 115.38, "p95_ms": 101.947}, "yolo11_640_fire_11class": {"avg_ms": 66.341, "count": 40, "max_ms": 84.546, "p95_ms": 83.539}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.287, "count": 40, "max_ms": 99.764, "p95_ms": 96.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.55, "count": 40, "max_ms": 166.527, "p95_ms": 155.275}, "read_interval": {"avg_ms": 766.488, "count": 39, "max_ms": 15827.346, "p95_ms": 8838.445}, "triton_wait": {"avg_ms": 78.929, "count": 40, "max_ms": 115.392, "p95_ms": 101.959}, "visualize": {"avg_ms": 4.946, "count": 40, "max_ms": 5.769, "p95_ms": 5.4}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 777}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5457.82, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:10:59+09:00", "last_error_summary": "[h264 @ 0x55f0a15f0fc0] error while decoding MB 114 25, bytestream -37 [in#0/rtsp @ 0x55f0a1593e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0a1593e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:10:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 437.635, "last_started_at": "2026-07-06T16:03:42+09:00", "pid": 15133, "process": {"exists": false, "pid": 15133}, "restart_count": 36, "restart_count_per_hour": 8, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.294, "sample_count": 2147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 93321, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:10:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T16:10:59+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-06T16:10:59+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-06T16:10:59.721250+09:00", "mtime_epoch": 1783321859.7212496, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:10:59+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1353.586, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.338, "pushed": 1.338, "read": 1.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.453, "count": 40, "max_ms": 8.69, "p95_ms": 7.528}, "gstreamer_push": {"avg_ms": 1.167, "count": 40, "max_ms": 2.793, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.938, "count": 40, "max_ms": 82.332, "p95_ms": 80.763}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.523, "count": 40, "max_ms": 115.38, "p95_ms": 101.947}, "yolo11_640_fire_11class": {"avg_ms": 66.341, "count": 40, "max_ms": 84.546, "p95_ms": 83.539}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.287, "count": 40, "max_ms": 99.764, "p95_ms": 96.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.55, "count": 40, "max_ms": 166.527, "p95_ms": 155.275}, "read_interval": {"avg_ms": 766.488, "count": 39, "max_ms": 15827.346, "p95_ms": 8838.445}, "triton_wait": {"avg_ms": 78.929, "count": 40, "max_ms": 115.392, "p95_ms": 101.959}, "visualize": {"avg_ms": 4.946, "count": 40, "max_ms": 5.769, "p95_ms": 5.4}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 777}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5654.832, "limit_mb": 8796093022207.996}, "detail": {"pid": 15606, "restart_count": 37, "start_count": 38}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:10:59+09:00", "last_error_summary": "[h264 @ 0x55f0a15f0fc0] error while decoding MB 114 25, bytestream -37 [in#0/rtsp @ 0x55f0a1593e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f0a1593e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:11:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:11:02+09:00", "pid": 15606, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.934, "name": "ffmpeg", "pid": 15606, "rss_mb": 194.934, "state": "S (sleeping)", "threads": 106}, "restart_count": 37, "restart_count_per_hour": 9, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.294, "sample_count": 2116, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 93321, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:10:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.269, "last_frame_pushed_at": "2026-07-06T16:10:59+09:00", "last_frame_read_age_sec": 4.421, "last_frame_read_at": "2026-07-06T16:10:59+09:00", "mjpeg": {"age_sec": 4.254, "exists": true, "mtime": "2026-07-06T16:10:59.877254+09:00", "mtime_epoch": 1783321859.8772538, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247583}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:11:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1353.586, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.598, "pushed": 0.598, "read": 0.598}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.298, "count": 13, "max_ms": 7.072, "p95_ms": 7.072}, "gstreamer_push": {"avg_ms": 1.076, "count": 13, "max_ms": 1.226, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.773, "count": 13, "max_ms": 78.631, "p95_ms": 78.631}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.711, "count": 13, "max_ms": 101.947, "p95_ms": 101.947}, "yolo11_640_fire_11class": {"avg_ms": 62.545, "count": 13, "max_ms": 79.136, "p95_ms": 79.136}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.819, "count": 13, "max_ms": 96.649, "p95_ms": 96.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.142, "count": 13, "max_ms": 155.275, "p95_ms": 155.275}, "read_interval": {"avg_ms": 2010.917, "count": 13, "max_ms": 15827.346, "p95_ms": 15827.346}, "triton_wait": {"avg_ms": 76.031, "count": 13, "max_ms": 101.959, "p95_ms": 101.959}, "visualize": {"avg_ms": 4.717, "count": 13, "max_ms": 5.769, "p95_ms": 5.769}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.725, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 913}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5576.828, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55acaade2ec0] error while decoding MB 47 33, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 171, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:13:35+09:00", "last_error_summary": "[h264 @ 0x55acaade2ec0] error while decoding MB 47 33, bytestream -7", "last_exit_at": "2026-07-06T16:10:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.937, "last_started_at": "2026-07-06T16:10:53+09:00", "pid": 15499, "process": {"exists": false, "pid": 15499}, "restart_count": 29, "restart_count_per_hour": 11, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.28, "sample_count": 2070, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 109648, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:13:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T16:13:35+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T16:13:35+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T16:13:35.929207+09:00", "mtime_epoch": 1783322015.929207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207811}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:13:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1492.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.548, "pushed": 7.548, "read": 7.548}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.001, "count": 226, "max_ms": 9.17, "p95_ms": 7.243}, "gstreamer_push": {"avg_ms": 1.206, "count": 226, "max_ms": 4.358, "p95_ms": 2.25}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.962, "count": 226, "max_ms": 81.32, "p95_ms": 70.763}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.708, "count": 226, "max_ms": 122.956, "p95_ms": 100.314}, "yolo11_640_fire_11class": {"avg_ms": 57.81, "count": 226, "max_ms": 88.86, "p95_ms": 76.443}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.445, "count": 226, "max_ms": 101.433, "p95_ms": 92.717}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.355, "count": 226, "max_ms": 177.689, "p95_ms": 154.291}, "read_interval": {"avg_ms": 132.915, "count": 226, "max_ms": 186.668, "p95_ms": 166.141}, "triton_wait": {"avg_ms": 70.79, "count": 226, "max_ms": 122.969, "p95_ms": 100.328}, "visualize": {"avg_ms": 5.168, "count": 226, "max_ms": 8.77, "p95_ms": 5.814}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 913}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5576.02, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:13:35+09:00", "last_error_summary": "[h264 @ 0x55acaade2ec0] error while decoding MB 47 33, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 171, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:13:35+09:00", "last_error_summary": "[h264 @ 0x55acaade2ec0] error while decoding MB 47 33, bytestream -7", "last_exit_at": "2026-07-06T16:13:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.039, "last_started_at": "2026-07-06T16:10:53+09:00", "pid": 15499, "process": {"exists": false, "pid": 15499}, "restart_count": 29, "restart_count_per_hour": 11, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.101, "sample_count": 2071, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 109649, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:13:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T16:13:36+09:00", "last_frame_read_age_sec": 0.153, "last_frame_read_at": "2026-07-06T16:13:35+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-06T16:13:35.929207+09:00", "mtime_epoch": 1783322015.929207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207811}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:13:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1492.914, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.557, "pushed": 7.557, "read": 7.523}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.001, "count": 226, "max_ms": 9.17, "p95_ms": 7.243}, "gstreamer_push": {"avg_ms": 1.206, "count": 226, "max_ms": 4.358, "p95_ms": 2.25}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.915, "count": 226, "max_ms": 81.32, "p95_ms": 70.763}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.702, "count": 226, "max_ms": 122.956, "p95_ms": 100.314}, "yolo11_640_fire_11class": {"avg_ms": 57.746, "count": 226, "max_ms": 88.86, "p95_ms": 76.255}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.347, "count": 226, "max_ms": 101.433, "p95_ms": 92.699}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.339, "count": 226, "max_ms": 177.689, "p95_ms": 154.291}, "read_interval": {"avg_ms": 132.798, "count": 225, "max_ms": 186.668, "p95_ms": 166.141}, "triton_wait": {"avg_ms": 70.785, "count": 226, "max_ms": 122.969, "p95_ms": 100.328}, "visualize": {"avg_ms": 5.168, "count": 226, "max_ms": 8.77, "p95_ms": 5.814}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 913}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5575.988, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 171, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:13:35+09:00", "last_error_summary": "[h264 @ 0x55acaade2ec0] error while decoding MB 47 33, bytestream -7", "last_exit_at": "2026-07-06T16:13:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.045, "last_started_at": "2026-07-06T16:10:53+09:00", "pid": 15499, "process": {"exists": false, "pid": 15499}, "restart_count": 29, "restart_count_per_hour": 11, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.101, "sample_count": 2071, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 109649, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:13:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T16:13:36+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-06T16:13:35+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-06T16:13:35.929207+09:00", "mtime_epoch": 1783322015.929207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207811}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:13:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1492.914, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.555, "pushed": 7.555, "read": 7.522}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.001, "count": 226, "max_ms": 9.17, "p95_ms": 7.243}, "gstreamer_push": {"avg_ms": 1.206, "count": 226, "max_ms": 4.358, "p95_ms": 2.25}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.915, "count": 226, "max_ms": 81.32, "p95_ms": 70.763}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.702, "count": 226, "max_ms": 122.956, "p95_ms": 100.314}, "yolo11_640_fire_11class": {"avg_ms": 57.746, "count": 226, "max_ms": 88.86, "p95_ms": 76.255}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.347, "count": 226, "max_ms": 101.433, "p95_ms": 92.699}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.339, "count": 226, "max_ms": 177.689, "p95_ms": 154.291}, "read_interval": {"avg_ms": 132.798, "count": 225, "max_ms": 186.668, "p95_ms": 166.141}, "triton_wait": {"avg_ms": 70.785, "count": 226, "max_ms": 122.969, "p95_ms": 100.328}, "visualize": {"avg_ms": 5.168, "count": 226, "max_ms": 8.77, "p95_ms": 5.814}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 913}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5681.004, "limit_mb": 8796093022207.996}, "detail": {"pid": 15758, "restart_count": 30, "start_count": 31}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 171, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:13:35+09:00", "last_error_summary": "[h264 @ 0x55acaade2ec0] error while decoding MB 47 33, bytestream -7", "last_exit_at": "2026-07-06T16:13:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:13:38+09:00", "pid": 15758, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.258, "name": "ffmpeg", "pid": 15758, "rss_mb": 136.258, "state": "S (sleeping)", "threads": 106}, "restart_count": 30, "restart_count_per_hour": 12, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.71, "sample_count": 2040, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 109649, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:13:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.39, "last_frame_pushed_at": "2026-07-06T16:13:36+09:00", "last_frame_read_age_sec": 4.538, "last_frame_read_at": "2026-07-06T16:13:35+09:00", "mjpeg": {"age_sec": 4.375, "exists": true, "mtime": "2026-07-06T16:13:36.081211+09:00", "mtime_epoch": 1783322016.0812109, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207771}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:13:40+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1492.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.564, "pushed": 6.564, "read": 6.53}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.024, "count": 196, "max_ms": 9.17, "p95_ms": 7.948}, "gstreamer_push": {"avg_ms": 1.191, "count": 196, "max_ms": 4.358, "p95_ms": 2.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.89, "count": 196, "max_ms": 77.549, "p95_ms": 69.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.611, "count": 196, "max_ms": 104.623, "p95_ms": 94.933}, "yolo11_640_fire_11class": {"avg_ms": 56.523, "count": 196, "max_ms": 85.63, "p95_ms": 74.061}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.465, "count": 196, "max_ms": 101.433, "p95_ms": 87.758}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.129, "count": 196, "max_ms": 174.329, "p95_ms": 147.352}, "read_interval": {"avg_ms": 130.578, "count": 195, "max_ms": 186.093, "p95_ms": 157.73}, "triton_wait": {"avg_ms": 68.689, "count": 196, "max_ms": 104.636, "p95_ms": 94.946}, "visualize": {"avg_ms": 5.182, "count": 196, "max_ms": 8.77, "p95_ms": 5.874}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 786}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5503.645, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55be8a09bd80] error while decoding MB 92 36, bytestream -21 [in#0/rtsp @ 0x55be89e9ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be89e9ee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:14:01+09:00", "last_error_summary": "[h264 @ 0x55be8a09bd80] error while decoding MB 92 36, bytestream -21 [in#0/rtsp @ 0x55be89e9ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be89e9ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:11:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.93, "last_started_at": "2026-07-06T16:11:02+09:00", "pid": 15606, "process": {"exists": false, "pid": 15606}, "restart_count": 37, "restart_count_per_hour": 9, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.975, "sample_count": 1930, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 94390, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:14:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-06T16:14:00+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-06T16:14:00+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T16:14:01.005817+09:00", "mtime_epoch": 1783322041.0058167, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242974}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:14:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1351.566, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 134, "select_timeouts": 11, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.594, "pushed": 4.594, "read": 4.629}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.282, "count": 133, "max_ms": 8.614, "p95_ms": 7.757}, "gstreamer_push": {"avg_ms": 1.201, "count": 133, "max_ms": 3.902, "p95_ms": 2.446}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.773, "count": 133, "max_ms": 86.209, "p95_ms": 78.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.543, "count": 133, "max_ms": 129.49, "p95_ms": 109.119}, "yolo11_640_fire_11class": {"avg_ms": 64.895, "count": 133, "max_ms": 86.747, "p95_ms": 81.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.0, "count": 133, "max_ms": 106.836, "p95_ms": 99.904}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.32, "count": 133, "max_ms": 179.946, "p95_ms": 161.814}, "read_interval": {"avg_ms": 323.017, "count": 134, "max_ms": 14406.523, "p95_ms": 170.632}, "triton_wait": {"avg_ms": 80.577, "count": 133, "max_ms": 129.502, "p95_ms": 109.131}, "visualize": {"avg_ms": 5.023, "count": 133, "max_ms": 6.585, "p95_ms": 5.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.948, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 786}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5505.301, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:14:01+09:00", "last_error_summary": "[h264 @ 0x55be8a09bd80] error while decoding MB 92 36, bytestream -21 [in#0/rtsp @ 0x55be89e9ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be89e9ee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:14:01+09:00", "last_error_summary": "[h264 @ 0x55be8a09bd80] error while decoding MB 92 36, bytestream -21 [in#0/rtsp @ 0x55be89e9ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be89e9ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:14:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.988, "last_started_at": "2026-07-06T16:11:02+09:00", "pid": 15606, "process": {"exists": false, "pid": 15606}, "restart_count": 37, "restart_count_per_hour": 9, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.975, "sample_count": 1930, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 94391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:14:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:14:01+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T16:14:00+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T16:14:01.005817+09:00", "mtime_epoch": 1783322041.0058167, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242974}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:14:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1351.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 134, "select_timeouts": 11, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.62, "pushed": 4.62, "read": 4.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.289, "count": 134, "max_ms": 8.614, "p95_ms": 7.757}, "gstreamer_push": {"avg_ms": 1.2, "count": 134, "max_ms": 3.902, "p95_ms": 2.446}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.679, "count": 134, "max_ms": 86.209, "p95_ms": 78.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.399, "count": 134, "max_ms": 129.49, "p95_ms": 109.119}, "yolo11_640_fire_11class": {"avg_ms": 64.831, "count": 134, "max_ms": 86.747, "p95_ms": 81.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.873, "count": 134, "max_ms": 106.836, "p95_ms": 99.904}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.242, "count": 134, "max_ms": 179.946, "p95_ms": 161.814}, "read_interval": {"avg_ms": 323.017, "count": 134, "max_ms": 14406.523, "p95_ms": 170.632}, "triton_wait": {"avg_ms": 80.426, "count": 134, "max_ms": 129.502, "p95_ms": 109.131}, "visualize": {"avg_ms": 5.037, "count": 134, "max_ms": 6.831, "p95_ms": 5.631}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.006, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 786}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5497.629, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:14:01+09:00", "last_error_summary": "[h264 @ 0x55be8a09bd80] error while decoding MB 92 36, bytestream -21 [in#0/rtsp @ 0x55be89e9ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be89e9ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:14:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.993, "last_started_at": "2026-07-06T16:11:02+09:00", "pid": 15606, "process": {"exists": false, "pid": 15606}, "restart_count": 37, "restart_count_per_hour": 9, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.975, "sample_count": 1930, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 94391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:14:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T16:14:01+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T16:14:00+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-06T16:14:01.005817+09:00", "mtime_epoch": 1783322041.0058167, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242974}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:14:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1351.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 134, "select_timeouts": 11, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.619, "pushed": 4.619, "read": 4.619}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.289, "count": 134, "max_ms": 8.614, "p95_ms": 7.757}, "gstreamer_push": {"avg_ms": 1.2, "count": 134, "max_ms": 3.902, "p95_ms": 2.446}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.679, "count": 134, "max_ms": 86.209, "p95_ms": 78.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.399, "count": 134, "max_ms": 129.49, "p95_ms": 109.119}, "yolo11_640_fire_11class": {"avg_ms": 64.831, "count": 134, "max_ms": 86.747, "p95_ms": 81.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.873, "count": 134, "max_ms": 106.836, "p95_ms": 99.904}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.242, "count": 134, "max_ms": 179.946, "p95_ms": 161.814}, "read_interval": {"avg_ms": 323.017, "count": 134, "max_ms": 14406.523, "p95_ms": 170.632}, "triton_wait": {"avg_ms": 80.426, "count": 134, "max_ms": 129.502, "p95_ms": 109.131}, "visualize": {"avg_ms": 5.037, "count": 134, "max_ms": 6.831, "p95_ms": 5.631}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.011, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 786}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5523.73, "limit_mb": 8796093022207.996}, "detail": {"pid": 15865, "restart_count": 38, "start_count": 39}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:14:01+09:00", "last_error_summary": "[h264 @ 0x55be8a09bd80] error while decoding MB 92 36, bytestream -21 [in#0/rtsp @ 0x55be89e9ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be89e9ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:14:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:14:03+09:00", "pid": 15865, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.578, "name": "ffmpeg", "pid": 15865, "rss_mb": 58.578, "state": "S (sleeping)", "threads": 1}, "restart_count": 38, "restart_count_per_hour": 10, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.08, "sample_count": 1886, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 94391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:14:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.337, "last_frame_pushed_at": "2026-07-06T16:14:01+09:00", "last_frame_read_age_sec": 4.461, "last_frame_read_at": "2026-07-06T16:14:00+09:00", "mjpeg": {"age_sec": 4.317, "exists": true, "mtime": "2026-07-06T16:14:01.133820+09:00", "mtime_epoch": 1783322041.1338198, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238882}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:14:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1351.566, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 133, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.741, "pushed": 5.741, "read": 5.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.303, "count": 133, "max_ms": 8.614, "p95_ms": 7.757}, "gstreamer_push": {"avg_ms": 1.198, "count": 133, "max_ms": 3.902, "p95_ms": 2.446}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.791, "count": 133, "max_ms": 86.209, "p95_ms": 78.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.577, "count": 133, "max_ms": 129.49, "p95_ms": 109.119}, "yolo11_640_fire_11class": {"avg_ms": 64.919, "count": 133, "max_ms": 86.747, "p95_ms": 81.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.018, "count": 133, "max_ms": 106.836, "p95_ms": 99.904}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.714, "count": 133, "max_ms": 179.946, "p95_ms": 161.814}, "read_interval": {"avg_ms": 217.126, "count": 133, "max_ms": 10171.708, "p95_ms": 170.304}, "triton_wait": {"avg_ms": 80.612, "count": 133, "max_ms": 129.502, "p95_ms": 109.131}, "visualize": {"avg_ms": 5.047, "count": 133, "max_ms": 6.831, "p95_ms": 5.631}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.167, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 794}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5530.078, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c3003f3900] error while decoding MB 62 47, bytestream -31 [in#0/rtsp @ 0x55c30023ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c30023ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:16:34+09:00", "last_error_summary": "[h264 @ 0x55c3003f3900] error while decoding MB 62 47, bytestream -31 [in#0/rtsp @ 0x55c30023ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c30023ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:14:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 151.377, "last_started_at": "2026-07-06T16:14:03+09:00", "pid": 15865, "process": {"exists": false, "pid": 15865}, "restart_count": 38, "restart_count_per_hour": 10, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.399, "sample_count": 1818, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:16:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-06T16:16:34+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T16:16:34+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T16:16:34.773431+09:00", "mtime_epoch": 1783322194.7734306, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243396}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:16:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1351.953, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 82, "pushed_frames": 82, "read_frames": 83, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.442, "pushed": 3.442, "read": 3.484}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.403, "count": 82, "max_ms": 8.602, "p95_ms": 7.432}, "gstreamer_push": {"avg_ms": 1.192, "count": 82, "max_ms": 3.834, "p95_ms": 2.464}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.725, "count": 82, "max_ms": 94.821, "p95_ms": 80.166}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.037, "count": 82, "max_ms": 136.13, "p95_ms": 104.834}, "yolo11_640_fire_11class": {"avg_ms": 71.102, "count": 82, "max_ms": 97.956, "p95_ms": 86.643}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.319, "count": 82, "max_ms": 108.722, "p95_ms": 99.22}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.052, "count": 82, "max_ms": 190.106, "p95_ms": 157.35}, "read_interval": {"avg_ms": 390.647, "count": 83, "max_ms": 11782.856, "p95_ms": 182.697}, "triton_wait": {"avg_ms": 85.3, "count": 82, "max_ms": 136.143, "p95_ms": 104.845}, "visualize": {"avg_ms": 5.15, "count": 82, "max_ms": 6.322, "p95_ms": 5.606}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.824, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 794}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5525.363, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:16:34+09:00", "last_error_summary": "[h264 @ 0x55c3003f3900] error while decoding MB 62 47, bytestream -31 [in#0/rtsp @ 0x55c30023ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c30023ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:16:34+09:00", "last_error_summary": "[h264 @ 0x55c3003f3900] error while decoding MB 62 47, bytestream -31 [in#0/rtsp @ 0x55c30023ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c30023ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:16:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 151.466, "last_started_at": "2026-07-06T16:14:03+09:00", "pid": 15865, "process": {"exists": false, "pid": 15865}, "restart_count": 38, "restart_count_per_hour": 10, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.399, "sample_count": 1819, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95360, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:16:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:16:34+09:00", "last_frame_read_age_sec": 0.15, "last_frame_read_at": "2026-07-06T16:16:34+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T16:16:34.773431+09:00", "mtime_epoch": 1783322194.7734306, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243396}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:16:34+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1348.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.471, "pushed": 3.471, "read": 3.471}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.4, "count": 83, "max_ms": 8.602, "p95_ms": 7.432}, "gstreamer_push": {"avg_ms": 1.224, "count": 83, "max_ms": 3.834, "p95_ms": 2.486}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.857, "count": 83, "max_ms": 94.821, "p95_ms": 80.166}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.113, "count": 83, "max_ms": 136.13, "p95_ms": 104.834}, "yolo11_640_fire_11class": {"avg_ms": 71.121, "count": 83, "max_ms": 97.956, "p95_ms": 86.643}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.419, "count": 83, "max_ms": 108.722, "p95_ms": 99.22}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.09, "count": 83, "max_ms": 190.106, "p95_ms": 157.35}, "read_interval": {"avg_ms": 390.647, "count": 83, "max_ms": 11782.856, "p95_ms": 182.697}, "triton_wait": {"avg_ms": 85.361, "count": 83, "max_ms": 136.143, "p95_ms": 104.845}, "visualize": {"avg_ms": 5.147, "count": 83, "max_ms": 6.322, "p95_ms": 5.606}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.911, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 794}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5525.348, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:16:34+09:00", "last_error_summary": "[h264 @ 0x55c3003f3900] error while decoding MB 62 47, bytestream -31 [in#0/rtsp @ 0x55c30023ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c30023ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:16:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 151.47, "last_started_at": "2026-07-06T16:14:03+09:00", "pid": 15865, "process": {"exists": false, "pid": 15865}, "restart_count": 38, "restart_count_per_hour": 10, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.399, "sample_count": 1819, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95360, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:16:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T16:16:34+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-06T16:16:34+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-06T16:16:34.773431+09:00", "mtime_epoch": 1783322194.7734306, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243396}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:16:34+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1348.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.47, "pushed": 3.47, "read": 3.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.4, "count": 83, "max_ms": 8.602, "p95_ms": 7.432}, "gstreamer_push": {"avg_ms": 1.224, "count": 83, "max_ms": 3.834, "p95_ms": 2.486}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.857, "count": 83, "max_ms": 94.821, "p95_ms": 80.166}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.113, "count": 83, "max_ms": 136.13, "p95_ms": 104.834}, "yolo11_640_fire_11class": {"avg_ms": 71.121, "count": 83, "max_ms": 97.956, "p95_ms": 86.643}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.419, "count": 83, "max_ms": 108.722, "p95_ms": 99.22}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.09, "count": 83, "max_ms": 190.106, "p95_ms": 157.35}, "read_interval": {"avg_ms": 390.647, "count": 83, "max_ms": 11782.856, "p95_ms": 182.697}, "triton_wait": {"avg_ms": 85.361, "count": 83, "max_ms": 136.143, "p95_ms": 104.845}, "visualize": {"avg_ms": 5.147, "count": 83, "max_ms": 6.322, "p95_ms": 5.606}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.916, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 794}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5546.117, "limit_mb": 8796093022207.996}, "detail": {"pid": 16017, "restart_count": 39, "start_count": 40}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:16:34+09:00", "last_error_summary": "[h264 @ 0x55c3003f3900] error while decoding MB 62 47, bytestream -31 [in#0/rtsp @ 0x55c30023ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c30023ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:16:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:16:37+09:00", "pid": 16017, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.008, "name": "ffmpeg", "pid": 16017, "rss_mb": 58.008, "state": "S (sleeping)", "threads": 1}, "restart_count": 39, "restart_count_per_hour": 11, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.265, "sample_count": 1788, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95360, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:16:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.552, "last_frame_pushed_at": "2026-07-06T16:16:34+09:00", "last_frame_read_age_sec": 4.698, "last_frame_read_at": "2026-07-06T16:16:34+09:00", "mjpeg": {"age_sec": 4.534, "exists": true, "mtime": "2026-07-06T16:16:34.929434+09:00", "mtime_epoch": 1783322194.929434, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243396}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:16:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1348.285, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.916, "pushed": 2.916, "read": 2.916}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.4, "count": 83, "max_ms": 8.602, "p95_ms": 7.432}, "gstreamer_push": {"avg_ms": 1.224, "count": 83, "max_ms": 3.834, "p95_ms": 2.486}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.857, "count": 83, "max_ms": 94.821, "p95_ms": 80.166}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.113, "count": 83, "max_ms": 136.13, "p95_ms": 104.834}, "yolo11_640_fire_11class": {"avg_ms": 71.121, "count": 83, "max_ms": 97.956, "p95_ms": 86.643}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.419, "count": 83, "max_ms": 108.722, "p95_ms": 99.22}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.09, "count": 83, "max_ms": 190.106, "p95_ms": 157.35}, "read_interval": {"avg_ms": 390.647, "count": 83, "max_ms": 11782.856, "p95_ms": 182.697}, "triton_wait": {"avg_ms": 85.361, "count": 83, "max_ms": 136.143, "p95_ms": 104.845}, "visualize": {"avg_ms": 5.147, "count": 83, "max_ms": 6.322, "p95_ms": 5.606}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.459, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 924}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5571.887, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5615172b4740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5615172b4740] error while decoding MB 0 41, bytestream 1552 [h264 @ 0x56151663e380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56151663e380] error while decoding MB 0 4, bytestream 344 [h264 @ 0x561517116a40] error while decoding MB 40 19, bytestream -24 [in#0/rtsp @ 0x5615164f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5615164f7e00] Error retrieving a packet from dem", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 174, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:17:09+09:00", "last_error_summary": "[h264 @ 0x5615172b4740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5615172b4740] error while decoding MB 0 41, bytestream 1552 [h264 @ 0x56151663e380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56151663e380] error while decoding MB 0 4, bytestream 344 [h264 @ 0x561517116a40] error while decoding MB 40 19, bytestream -24 [in#0/rtsp @ 0x5615164f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5615164f7e00] Error retrieving a packet from dem", "last_exit_at": "2026-07-06T16:13:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 211.354, "last_started_at": "2026-07-06T16:13:38+09:00", "pid": 15758, "process": {"exists": false, "pid": 15758}, "restart_count": 30, "restart_count_per_hour": 11, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.284, "sample_count": 1921, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 110969, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:17:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T16:17:09+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T16:17:09+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T16:17:09.766227+09:00", "mtime_epoch": 1783322229.766227, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:17:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1491.238, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 85, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.837, "pushed": 2.837, "read": 2.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.815, "count": 85, "max_ms": 8.281, "p95_ms": 7.013}, "gstreamer_push": {"avg_ms": 1.17, "count": 85, "max_ms": 5.25, "p95_ms": 1.263}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.128, "count": 85, "max_ms": 79.715, "p95_ms": 73.265}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.598, "count": 85, "max_ms": 105.152, "p95_ms": 97.075}, "yolo11_640_fire_11class": {"avg_ms": 58.08, "count": 85, "max_ms": 86.273, "p95_ms": 79.784}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.94, "count": 85, "max_ms": 104.815, "p95_ms": 91.107}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.059, "count": 85, "max_ms": 174.558, "p95_ms": 152.12}, "read_interval": {"avg_ms": 353.213, "count": 85, "max_ms": 18873.224, "p95_ms": 165.024}, "triton_wait": {"avg_ms": 70.733, "count": 85, "max_ms": 105.162, "p95_ms": 98.829}, "visualize": {"avg_ms": 5.017, "count": 85, "max_ms": 6.829, "p95_ms": 5.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 924}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5571.086, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:17:09+09:00", "last_error_summary": "[h264 @ 0x5615172b4740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5615172b4740] error while decoding MB 0 41, bytestream 1552 [h264 @ 0x56151663e380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56151663e380] error while decoding MB 0 4, bytestream 344 [h264 @ 0x561517116a40] error while decoding MB 40 19, bytestream -24 [in#0/rtsp @ 0x5615164f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5615164f7e00] Error retrieving a packet from dem", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 174, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:17:09+09:00", "last_error_summary": "[h264 @ 0x5615172b4740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5615172b4740] error while decoding MB 0 41, bytestream 1552 [h264 @ 0x56151663e380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56151663e380] error while decoding MB 0 4, bytestream 344 [h264 @ 0x561517116a40] error while decoding MB 40 19, bytestream -24 [in#0/rtsp @ 0x5615164f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5615164f7e00] Error retrieving a packet from dem", "last_exit_at": "2026-07-06T16:17:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 211.46, "last_started_at": "2026-07-06T16:13:38+09:00", "pid": 15758, "process": {"exists": false, "pid": 15758}, "restart_count": 30, "restart_count_per_hour": 11, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.284, "sample_count": 1922, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 110970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:17:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:17:09+09:00", "last_frame_read_age_sec": 0.158, "last_frame_read_at": "2026-07-06T16:17:09+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-06T16:17:09.766227+09:00", "mtime_epoch": 1783322229.766227, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:17:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1491.238, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.839, "pushed": 2.839, "read": 2.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.812, "count": 85, "max_ms": 8.281, "p95_ms": 7.013}, "gstreamer_push": {"avg_ms": 1.168, "count": 85, "max_ms": 5.25, "p95_ms": 1.263}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.311, "count": 85, "max_ms": 79.715, "p95_ms": 73.265}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.997, "count": 85, "max_ms": 105.152, "p95_ms": 98.566}, "yolo11_640_fire_11class": {"avg_ms": 58.216, "count": 85, "max_ms": 86.273, "p95_ms": 79.784}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.41, "count": 85, "max_ms": 104.815, "p95_ms": 91.517}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.45, "count": 85, "max_ms": 174.558, "p95_ms": 152.709}, "read_interval": {"avg_ms": 355.889, "count": 84, "max_ms": 18873.224, "p95_ms": 165.024}, "triton_wait": {"avg_ms": 71.132, "count": 85, "max_ms": 105.162, "p95_ms": 100.112}, "visualize": {"avg_ms": 5.013, "count": 85, "max_ms": 6.829, "p95_ms": 5.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 924}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5571.027, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 174, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:17:09+09:00", "last_error_summary": "[h264 @ 0x5615172b4740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5615172b4740] error while decoding MB 0 41, bytestream 1552 [h264 @ 0x56151663e380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56151663e380] error while decoding MB 0 4, bytestream 344 [h264 @ 0x561517116a40] error while decoding MB 40 19, bytestream -24 [in#0/rtsp @ 0x5615164f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5615164f7e00] Error retrieving a packet from dem", "last_exit_at": "2026-07-06T16:17:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 211.465, "last_started_at": "2026-07-06T16:13:38+09:00", "pid": 15758, "process": {"exists": false, "pid": 15758}, "restart_count": 30, "restart_count_per_hour": 11, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.284, "sample_count": 1922, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 110970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:17:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T16:17:09+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-06T16:17:09+09:00", "mjpeg": {"age_sec": 0.154, "exists": true, "mtime": "2026-07-06T16:17:09.766227+09:00", "mtime_epoch": 1783322229.766227, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:17:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1491.238, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.838, "pushed": 2.838, "read": 2.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.812, "count": 85, "max_ms": 8.281, "p95_ms": 7.013}, "gstreamer_push": {"avg_ms": 1.168, "count": 85, "max_ms": 5.25, "p95_ms": 1.263}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.311, "count": 85, "max_ms": 79.715, "p95_ms": 73.265}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.997, "count": 85, "max_ms": 105.152, "p95_ms": 98.566}, "yolo11_640_fire_11class": {"avg_ms": 58.216, "count": 85, "max_ms": 86.273, "p95_ms": 79.784}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.41, "count": 85, "max_ms": 104.815, "p95_ms": 91.517}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.45, "count": 85, "max_ms": 174.558, "p95_ms": 152.709}, "read_interval": {"avg_ms": 355.889, "count": 84, "max_ms": 18873.224, "p95_ms": 165.024}, "triton_wait": {"avg_ms": 71.132, "count": 85, "max_ms": 105.162, "p95_ms": 100.112}, "visualize": {"avg_ms": 5.013, "count": 85, "max_ms": 6.829, "p95_ms": 5.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 924}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5675.512, "limit_mb": 8796093022207.996}, "detail": {"pid": 16169, "restart_count": 31, "start_count": 32}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 174, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:17:09+09:00", "last_error_summary": "[h264 @ 0x5615172b4740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5615172b4740] error while decoding MB 0 41, bytestream 1552 [h264 @ 0x56151663e380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56151663e380] error while decoding MB 0 4, bytestream 344 [h264 @ 0x561517116a40] error while decoding MB 40 19, bytestream -24 [in#0/rtsp @ 0x5615164f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5615164f7e00] Error retrieving a packet from dem", "last_exit_at": "2026-07-06T16:17:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:17:12+09:00", "pid": 16169, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.465, "name": "ffmpeg", "pid": 16169, "rss_mb": 136.465, "state": "S (sleeping)", "threads": 106}, "restart_count": 31, "restart_count_per_hour": 12, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.12, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 110970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:17:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.352, "last_frame_pushed_at": "2026-07-06T16:17:09+09:00", "last_frame_read_age_sec": 4.506, "last_frame_read_at": "2026-07-06T16:17:09+09:00", "mjpeg": {"age_sec": 4.333, "exists": true, "mtime": "2026-07-06T16:17:09.930231+09:00", "mtime_epoch": 1783322229.9302306, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200229}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:17:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1491.238, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.704, "pushed": 1.704, "read": 1.671}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.794, "count": 51, "max_ms": 7.448, "p95_ms": 7.048}, "gstreamer_push": {"avg_ms": 1.213, "count": 51, "max_ms": 5.25, "p95_ms": 1.37}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.398, "count": 51, "max_ms": 79.715, "p95_ms": 76.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.631, "count": 51, "max_ms": 105.152, "p95_ms": 101.026}, "yolo11_640_fire_11class": {"avg_ms": 60.923, "count": 51, "max_ms": 83.892, "p95_ms": 83.032}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.313, "count": 51, "max_ms": 104.815, "p95_ms": 101.014}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.116, "count": 51, "max_ms": 174.558, "p95_ms": 154.942}, "read_interval": {"avg_ms": 511.446, "count": 50, "max_ms": 18873.224, "p95_ms": 170.679}, "triton_wait": {"avg_ms": 74.515, "count": 51, "max_ms": 105.162, "p95_ms": 102.72}, "visualize": {"avg_ms": 4.923, "count": 51, "max_ms": 6.209, "p95_ms": 5.448}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 798}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5505.645, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5640c6e57340] error while decoding MB 31 49, bytestream -11 [in#0/rtsp @ 0x5640c6147e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5640c6147e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:18:12+09:00", "last_error_summary": "[h264 @ 0x5640c6e57340] error while decoding MB 31 49, bytestream -11 [in#0/rtsp @ 0x5640c6147e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5640c6147e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:16:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 95.226, "last_started_at": "2026-07-06T16:16:37+09:00", "pid": 16017, "process": {"exists": false, "pid": 16017}, "restart_count": 39, "restart_count_per_hour": 11, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.71, "sample_count": 1640, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-06T16:18:12+09:00", "last_frame_read_age_sec": 0.075, "last_frame_read_at": "2026-07-06T16:18:12+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-06T16:18:12.627637+09:00", "mtime_epoch": 1783322292.627637, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222183}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:18:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1355.023, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.677, "pushed": 0.677, "read": 0.733}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.685, "count": 12, "max_ms": 6.523, "p95_ms": 6.523}, "gstreamer_push": {"avg_ms": 1.041, "count": 12, "max_ms": 1.254, "p95_ms": 1.254}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.249, "count": 12, "max_ms": 74.476, "p95_ms": 74.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.718, "count": 12, "max_ms": 99.483, "p95_ms": 99.483}, "yolo11_640_fire_11class": {"avg_ms": 60.633, "count": 12, "max_ms": 75.785, "p95_ms": 75.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.65, "count": 12, "max_ms": 91.214, "p95_ms": 91.214}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.952, "count": 12, "max_ms": 152.572, "p95_ms": 152.572}, "read_interval": {"avg_ms": 2529.006, "count": 13, "max_ms": 16256.012, "p95_ms": 16256.012}, "triton_wait": {"avg_ms": 74.729, "count": 12, "max_ms": 99.496, "p95_ms": 99.496}, "visualize": {"avg_ms": 4.695, "count": 12, "max_ms": 5.387, "p95_ms": 5.387}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.735, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 798}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5504.531, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:18:12+09:00", "last_error_summary": "[h264 @ 0x5640c6e57340] error while decoding MB 31 49, bytestream -11 [in#0/rtsp @ 0x5640c6147e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5640c6147e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:18:12+09:00", "last_error_summary": "[h264 @ 0x5640c6e57340] error while decoding MB 31 49, bytestream -11 [in#0/rtsp @ 0x5640c6147e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5640c6147e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:18:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 95.278, "last_started_at": "2026-07-06T16:16:37+09:00", "pid": 16017, "process": {"exists": false, "pid": 16017}, "restart_count": 39, "restart_count_per_hour": 11, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.71, "sample_count": 1641, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:18:12+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-06T16:18:12+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T16:18:12.627637+09:00", "mtime_epoch": 1783322292.627637, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222183}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:18:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1355.023, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.611, "count": 13, "max_ms": 6.523, "p95_ms": 6.523}, "gstreamer_push": {"avg_ms": 1.043, "count": 13, "max_ms": 1.254, "p95_ms": 1.254}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.301, "count": 13, "max_ms": 74.476, "p95_ms": 74.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.263, "count": 13, "max_ms": 99.483, "p95_ms": 99.483}, "yolo11_640_fire_11class": {"avg_ms": 60.088, "count": 13, "max_ms": 75.785, "p95_ms": 75.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.837, "count": 13, "max_ms": 91.214, "p95_ms": 91.214}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.041, "count": 13, "max_ms": 152.572, "p95_ms": 152.572}, "read_interval": {"avg_ms": 2529.006, "count": 13, "max_ms": 16256.012, "p95_ms": 16256.012}, "triton_wait": {"avg_ms": 74.274, "count": 13, "max_ms": 99.496, "p95_ms": 99.496}, "visualize": {"avg_ms": 4.717, "count": 13, "max_ms": 5.387, "p95_ms": 5.387}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.786, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 798}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5504.523, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:18:12+09:00", "last_error_summary": "[h264 @ 0x5640c6e57340] error while decoding MB 31 49, bytestream -11 [in#0/rtsp @ 0x5640c6147e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5640c6147e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:18:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 95.282, "last_started_at": "2026-07-06T16:16:37+09:00", "pid": 16017, "process": {"exists": false, "pid": 16017}, "restart_count": 39, "restart_count_per_hour": 11, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.71, "sample_count": 1641, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T16:18:12+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T16:18:12+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T16:18:12.627637+09:00", "mtime_epoch": 1783322292.627637, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222183}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:18:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1355.023, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.611, "count": 13, "max_ms": 6.523, "p95_ms": 6.523}, "gstreamer_push": {"avg_ms": 1.043, "count": 13, "max_ms": 1.254, "p95_ms": 1.254}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.301, "count": 13, "max_ms": 74.476, "p95_ms": 74.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.263, "count": 13, "max_ms": 99.483, "p95_ms": 99.483}, "yolo11_640_fire_11class": {"avg_ms": 60.088, "count": 13, "max_ms": 75.785, "p95_ms": 75.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.837, "count": 13, "max_ms": 91.214, "p95_ms": 91.214}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.041, "count": 13, "max_ms": 152.572, "p95_ms": 152.572}, "read_interval": {"avg_ms": 2529.006, "count": 13, "max_ms": 16256.012, "p95_ms": 16256.012}, "triton_wait": {"avg_ms": 74.274, "count": 13, "max_ms": 99.496, "p95_ms": 99.496}, "visualize": {"avg_ms": 4.717, "count": 13, "max_ms": 5.387, "p95_ms": 5.387}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.79, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 798}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5531.496, "limit_mb": 8796093022207.996}, "detail": {"pid": 16276, "restart_count": 40, "start_count": 41}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:18:12+09:00", "last_error_summary": "[h264 @ 0x5640c6e57340] error while decoding MB 31 49, bytestream -11 [in#0/rtsp @ 0x5640c6147e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5640c6147e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:18:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:18:15+09:00", "pid": 16276, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.301, "name": "ffmpeg", "pid": 16276, "rss_mb": 58.301, "state": "S (sleeping)", "threads": 1}, "restart_count": 40, "restart_count_per_hour": 12, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.697, "sample_count": 1606, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 95859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.7, "last_frame_pushed_at": "2026-07-06T16:18:12+09:00", "last_frame_read_age_sec": 4.823, "last_frame_read_at": "2026-07-06T16:18:12+09:00", "mjpeg": {"age_sec": 4.681, "exists": true, "mtime": "2026-07-06T16:18:12.755640+09:00", "mtime_epoch": 1783322292.7556398, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221978}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:18:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1355.023, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.578, "pushed": 0.578, "read": 0.578}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.611, "count": 13, "max_ms": 6.523, "p95_ms": 6.523}, "gstreamer_push": {"avg_ms": 1.043, "count": 13, "max_ms": 1.254, "p95_ms": 1.254}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.301, "count": 13, "max_ms": 74.476, "p95_ms": 74.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.263, "count": 13, "max_ms": 99.483, "p95_ms": 99.483}, "yolo11_640_fire_11class": {"avg_ms": 60.088, "count": 13, "max_ms": 75.785, "p95_ms": 75.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.837, "count": 13, "max_ms": 91.214, "p95_ms": 91.214}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.041, "count": 13, "max_ms": 152.572, "p95_ms": 152.572}, "read_interval": {"avg_ms": 2529.006, "count": 13, "max_ms": 16256.012, "p95_ms": 16256.012}, "triton_wait": {"avg_ms": 74.274, "count": 13, "max_ms": 99.496, "p95_ms": 99.496}, "visualize": {"avg_ms": 4.717, "count": 13, "max_ms": 5.387, "p95_ms": 5.387}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.482, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 803}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5506.941, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5646db3bb6c0] error while decoding MB 61 22, bytestream -11 [in#0/rtsp @ 0x5646da844e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646da844e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:19:57+09:00", "last_error_summary": "[h264 @ 0x5646db3bb6c0] error while decoding MB 61 22, bytestream -11 [in#0/rtsp @ 0x5646da844e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646da844e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:18:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 101.772, "last_started_at": "2026-07-06T16:18:15+09:00", "pid": 16276, "process": {"exists": false, "pid": 16276}, "restart_count": 40, "restart_count_per_hour": 11, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.31, "sample_count": 1623, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 96397, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:19:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T16:19:57+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T16:19:57+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-06T16:19:57.153930+09:00", "mtime_epoch": 1783322397.1539302, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239716}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:19:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1354.949, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.433, "pushed": 0.433, "read": 0.433}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.358, "count": 12, "max_ms": 7.981, "p95_ms": 7.981}, "gstreamer_push": {"avg_ms": 1.176, "count": 13, "max_ms": 2.324, "p95_ms": 2.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.468, "count": 12, "max_ms": 64.381, "p95_ms": 64.381}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.905, "count": 12, "max_ms": 103.19, "p95_ms": 103.19}, "yolo11_640_fire_11class": {"avg_ms": 56.663, "count": 12, "max_ms": 68.536, "p95_ms": 68.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.063, "count": 12, "max_ms": 84.922, "p95_ms": 84.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.225, "count": 13, "max_ms": 168.383, "p95_ms": 168.383}, "read_interval": {"avg_ms": 2310.997, "count": 13, "max_ms": 16446.982, "p95_ms": 16446.982}, "triton_wait": {"avg_ms": 68.917, "count": 12, "max_ms": 103.202, "p95_ms": 103.202}, "visualize": {"avg_ms": 5.079, "count": 12, "max_ms": 6.259, "p95_ms": 6.259}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 803}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5506.051, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:19:57+09:00", "last_error_summary": "[h264 @ 0x5646db3bb6c0] error while decoding MB 61 22, bytestream -11 [in#0/rtsp @ 0x5646da844e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646da844e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:19:57+09:00", "last_error_summary": "[h264 @ 0x5646db3bb6c0] error while decoding MB 61 22, bytestream -11 [in#0/rtsp @ 0x5646da844e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646da844e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:19:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 101.85, "last_started_at": "2026-07-06T16:18:15+09:00", "pid": 16276, "process": {"exists": false, "pid": 16276}, "restart_count": 40, "restart_count_per_hour": 11, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.31, "sample_count": 1623, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 96398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:19:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T16:19:57+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T16:19:57+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-06T16:19:57.153930+09:00", "mtime_epoch": 1783322397.1539302, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239716}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:19:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1354.949, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.724, "pushed": 0.724, "read": 0.724}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.46, "count": 13, "max_ms": 7.981, "p95_ms": 7.981}, "gstreamer_push": {"avg_ms": 1.178, "count": 13, "max_ms": 2.324, "p95_ms": 2.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.27, "count": 13, "max_ms": 64.381, "p95_ms": 64.381}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.026, "count": 13, "max_ms": 103.19, "p95_ms": 103.19}, "yolo11_640_fire_11class": {"avg_ms": 56.523, "count": 13, "max_ms": 68.536, "p95_ms": 68.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.967, "count": 13, "max_ms": 84.922, "p95_ms": 84.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.721, "count": 13, "max_ms": 168.383, "p95_ms": 168.383}, "read_interval": {"avg_ms": 2310.997, "count": 13, "max_ms": 16446.982, "p95_ms": 16446.982}, "triton_wait": {"avg_ms": 70.037, "count": 13, "max_ms": 103.202, "p95_ms": 103.202}, "visualize": {"avg_ms": 5.16, "count": 13, "max_ms": 6.259, "p95_ms": 6.259}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.962, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 803}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5506.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:19:57+09:00", "last_error_summary": "[h264 @ 0x5646db3bb6c0] error while decoding MB 61 22, bytestream -11 [in#0/rtsp @ 0x5646da844e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646da844e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:19:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 101.854, "last_started_at": "2026-07-06T16:18:15+09:00", "pid": 16276, "process": {"exists": false, "pid": 16276}, "restart_count": 40, "restart_count_per_hour": 11, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.31, "sample_count": 1623, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 96398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:19:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T16:19:57+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-06T16:19:57+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-06T16:19:57.153930+09:00", "mtime_epoch": 1783322397.1539302, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239716}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:19:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1354.949, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.724, "pushed": 0.724, "read": 0.724}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.46, "count": 13, "max_ms": 7.981, "p95_ms": 7.981}, "gstreamer_push": {"avg_ms": 1.178, "count": 13, "max_ms": 2.324, "p95_ms": 2.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.27, "count": 13, "max_ms": 64.381, "p95_ms": 64.381}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.026, "count": 13, "max_ms": 103.19, "p95_ms": 103.19}, "yolo11_640_fire_11class": {"avg_ms": 56.523, "count": 13, "max_ms": 68.536, "p95_ms": 68.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.967, "count": 13, "max_ms": 84.922, "p95_ms": 84.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.721, "count": 13, "max_ms": 168.383, "p95_ms": 168.383}, "read_interval": {"avg_ms": 2310.997, "count": 13, "max_ms": 16446.982, "p95_ms": 16446.982}, "triton_wait": {"avg_ms": 70.037, "count": 13, "max_ms": 103.202, "p95_ms": 103.202}, "visualize": {"avg_ms": 5.16, "count": 13, "max_ms": 6.259, "p95_ms": 6.259}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.966, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 803}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5535.953, "limit_mb": 8796093022207.996}, "detail": {"pid": 16428, "restart_count": 41, "start_count": 42}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:19:57+09:00", "last_error_summary": "[h264 @ 0x5646db3bb6c0] error while decoding MB 61 22, bytestream -11 [in#0/rtsp @ 0x5646da844e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646da844e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:19:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:19:59+09:00", "pid": 16428, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.328, "name": "ffmpeg", "pid": 16428, "rss_mb": 58.328, "state": "S (sleeping)", "threads": 1}, "restart_count": 41, "restart_count_per_hour": 12, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.305, "sample_count": 1593, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 96398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:19:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.112, "last_frame_pushed_at": "2026-07-06T16:19:57+09:00", "last_frame_read_age_sec": 4.253, "last_frame_read_at": "2026-07-06T16:19:57+09:00", "mjpeg": {"age_sec": 4.096, "exists": true, "mtime": "2026-07-06T16:19:57.297933+09:00", "mtime_epoch": 1783322397.2979333, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239198}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:20:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1354.949, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.589, "pushed": 0.589, "read": 0.589}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.46, "count": 13, "max_ms": 7.981, "p95_ms": 7.981}, "gstreamer_push": {"avg_ms": 1.178, "count": 13, "max_ms": 2.324, "p95_ms": 2.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.27, "count": 13, "max_ms": 64.381, "p95_ms": 64.381}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.026, "count": 13, "max_ms": 103.19, "p95_ms": 103.19}, "yolo11_640_fire_11class": {"avg_ms": 56.523, "count": 13, "max_ms": 68.536, "p95_ms": 68.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.967, "count": 13, "max_ms": 84.922, "p95_ms": 84.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.721, "count": 13, "max_ms": 168.383, "p95_ms": 168.383}, "read_interval": {"avg_ms": 2310.997, "count": 13, "max_ms": 16446.982, "p95_ms": 16446.982}, "triton_wait": {"avg_ms": 70.037, "count": 13, "max_ms": 103.202, "p95_ms": 103.202}, "visualize": {"avg_ms": 5.16, "count": 13, "max_ms": 6.259, "p95_ms": 6.259}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.07, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 938}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5577.156, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564203c40b80] error while decoding MB 72 21, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:20:50+09:00", "last_error_summary": "[h264 @ 0x564203c40b80] error while decoding MB 72 21, bytestream -7", "last_exit_at": "2026-07-06T16:17:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 218.436, "last_started_at": "2026-07-06T16:17:12+09:00", "pid": 16169, "process": {"exists": false, "pid": 16169}, "restart_count": 31, "restart_count_per_hour": 11, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.592, "sample_count": 2071, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 112612, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:20:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T16:20:50+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T16:20:50+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T16:20:50.659083+09:00", "mtime_epoch": 1783322450.6590827, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200565}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:20:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1494.113, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 211, "pushed_frames": 211, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.06, "pushed": 7.06, "read": 7.06}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.805, "count": 211, "max_ms": 9.119, "p95_ms": 7.583}, "gstreamer_push": {"avg_ms": 1.199, "count": 211, "max_ms": 3.816, "p95_ms": 1.45}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.318, "count": 211, "max_ms": 93.855, "p95_ms": 79.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.998, "count": 211, "max_ms": 126.238, "p95_ms": 107.345}, "yolo11_640_fire_11class": {"avg_ms": 66.619, "count": 211, "max_ms": 98.343, "p95_ms": 86.655}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.538, "count": 211, "max_ms": 113.298, "p95_ms": 102.785}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.228, "count": 211, "max_ms": 181.497, "p95_ms": 165.053}, "read_interval": {"avg_ms": 142.095, "count": 211, "max_ms": 190.457, "p95_ms": 175.177}, "triton_wait": {"avg_ms": 83.381, "count": 211, "max_ms": 126.25, "p95_ms": 107.357}, "visualize": {"avg_ms": 5.041, "count": 211, "max_ms": 8.469, "p95_ms": 6.197}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 938}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5576.273, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:20:50+09:00", "last_error_summary": "[h264 @ 0x564203c40b80] error while decoding MB 72 21, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:20:50+09:00", "last_error_summary": "[h264 @ 0x564203c40b80] error while decoding MB 72 21, bytestream -7", "last_exit_at": "2026-07-06T16:20:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 218.508, "last_started_at": "2026-07-06T16:17:12+09:00", "pid": 16169, "process": {"exists": false, "pid": 16169}, "restart_count": 31, "restart_count_per_hour": 11, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.592, "sample_count": 2071, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 112613, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:20:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T16:20:50+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-06T16:20:50+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T16:20:50.659083+09:00", "mtime_epoch": 1783322450.6590827, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200565}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:20:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1494.113, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.076, "pushed": 7.076, "read": 7.043}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.817, "count": 212, "max_ms": 9.119, "p95_ms": 7.59}, "gstreamer_push": {"avg_ms": 1.198, "count": 212, "max_ms": 3.816, "p95_ms": 1.45}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.311, "count": 212, "max_ms": 93.855, "p95_ms": 79.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.919, "count": 212, "max_ms": 126.238, "p95_ms": 107.345}, "yolo11_640_fire_11class": {"avg_ms": 66.581, "count": 212, "max_ms": 98.343, "p95_ms": 86.655}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.468, "count": 212, "max_ms": 113.298, "p95_ms": 102.785}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.16, "count": 212, "max_ms": 181.497, "p95_ms": 165.053}, "read_interval": {"avg_ms": 142.095, "count": 211, "max_ms": 190.457, "p95_ms": 175.177}, "triton_wait": {"avg_ms": 83.3, "count": 212, "max_ms": 126.25, "p95_ms": 107.357}, "visualize": {"avg_ms": 5.04, "count": 212, "max_ms": 8.469, "p95_ms": 6.197}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 938}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5576.266, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:20:50+09:00", "last_error_summary": "[h264 @ 0x564203c40b80] error while decoding MB 72 21, bytestream -7", "last_exit_at": "2026-07-06T16:20:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 218.514, "last_started_at": "2026-07-06T16:17:12+09:00", "pid": 16169, "process": {"exists": false, "pid": 16169}, "restart_count": 31, "restart_count_per_hour": 11, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.592, "sample_count": 2071, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 112613, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:20:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T16:20:50+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T16:20:50+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T16:20:50.659083+09:00", "mtime_epoch": 1783322450.6590827, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200565}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:20:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1494.113, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.075, "pushed": 7.075, "read": 7.042}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.817, "count": 211, "max_ms": 9.119, "p95_ms": 7.59}, "gstreamer_push": {"avg_ms": 1.198, "count": 212, "max_ms": 3.816, "p95_ms": 1.45}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.252, "count": 211, "max_ms": 93.855, "p95_ms": 79.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.861, "count": 211, "max_ms": 126.238, "p95_ms": 107.345}, "yolo11_640_fire_11class": {"avg_ms": 66.537, "count": 211, "max_ms": 98.343, "p95_ms": 86.655}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.393, "count": 211, "max_ms": 113.298, "p95_ms": 102.785}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.16, "count": 212, "max_ms": 181.497, "p95_ms": 165.053}, "read_interval": {"avg_ms": 142.095, "count": 211, "max_ms": 190.457, "p95_ms": 175.177}, "triton_wait": {"avg_ms": 83.244, "count": 211, "max_ms": 126.25, "p95_ms": 107.357}, "visualize": {"avg_ms": 5.039, "count": 211, "max_ms": 8.469, "p95_ms": 6.197}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 938}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5584.457, "limit_mb": 8796093022207.996}, "detail": {"pid": 16580, "restart_count": 32, "start_count": 33}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:20:50+09:00", "last_error_summary": "[h264 @ 0x564203c40b80] error while decoding MB 72 21, bytestream -7", "last_exit_at": "2026-07-06T16:20:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:20:52+09:00", "pid": 16580, "process": {"exists": true, "fd_count": 4, "hwm_mb": 43.574, "name": "ffmpeg", "pid": 16580, "rss_mb": 43.574, "state": "S (sleeping)", "threads": 1}, "restart_count": 32, "restart_count_per_hour": 12, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.218, "sample_count": 2047, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 112613, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:20:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.065, "last_frame_pushed_at": "2026-07-06T16:20:50+09:00", "last_frame_read_age_sec": 4.185, "last_frame_read_at": "2026-07-06T16:20:50+09:00", "mjpeg": {"age_sec": 4.049, "exists": true, "mtime": "2026-07-06T16:20:50.783085+09:00", "mtime_epoch": 1783322450.7830853, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200635}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:20:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1750.574, "name": "celery", "pid": 237, "rss_mb": 1494.113, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.168, "pushed": 6.168, "read": 6.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.778, "count": 184, "max_ms": 9.119, "p95_ms": 7.635}, "gstreamer_push": {"avg_ms": 1.194, "count": 185, "max_ms": 3.816, "p95_ms": 1.441}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.165, "count": 184, "max_ms": 87.194, "p95_ms": 79.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.664, "count": 184, "max_ms": 126.238, "p95_ms": 107.345}, "yolo11_640_fire_11class": {"avg_ms": 66.236, "count": 184, "max_ms": 96.605, "p95_ms": 85.975}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.107, "count": 184, "max_ms": 108.115, "p95_ms": 102.785}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.039, "count": 185, "max_ms": 181.497, "p95_ms": 163.701}, "read_interval": {"avg_ms": 141.073, "count": 184, "max_ms": 190.457, "p95_ms": 175.09}, "triton_wait": {"avg_ms": 83.021, "count": 184, "max_ms": 126.25, "p95_ms": 107.357}, "visualize": {"avg_ms": 5.008, "count": 184, "max_ms": 8.469, "p95_ms": 6.197}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 811}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5505.652, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55c36f79f200] missing picture in access unit with size 18 [h264 @ 0x55c36f9568c0] no frame! [in#0/rtsp @ 0x55c36f79ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c36f79ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c36f79f840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:22:39+09:00", "last_error_summary": "[NULL @ 0x55c36f79f200] missing picture in access unit with size 18 [h264 @ 0x55c36f9568c0] no frame! [in#0/rtsp @ 0x55c36f79ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c36f79ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c36f79f840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T16:19:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.962, "last_started_at": "2026-07-06T16:19:59+09:00", "pid": 16428, "process": {"exists": false, "pid": 16428}, "restart_count": 41, "restart_count_per_hour": 12, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.59, "sample_count": 1576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97417, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:22:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T16:22:39+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T16:22:39+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T16:22:39.305385+09:00", "mtime_epoch": 1783322559.305385, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254025}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:22:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1352.801, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 29, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.968, "pushed": 0.968, "read": 0.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.91, "count": 29, "max_ms": 9.627, "p95_ms": 8.172}, "gstreamer_push": {"avg_ms": 1.333, "count": 29, "max_ms": 3.616, "p95_ms": 1.579}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.399, "count": 29, "max_ms": 75.181, "p95_ms": 74.992}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.728, "count": 29, "max_ms": 104.667, "p95_ms": 100.215}, "yolo11_640_fire_11class": {"avg_ms": 64.668, "count": 29, "max_ms": 85.451, "p95_ms": 84.756}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.221, "count": 29, "max_ms": 102.287, "p95_ms": 97.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.934, "count": 29, "max_ms": 151.04, "p95_ms": 149.848}, "read_interval": {"avg_ms": 1034.865, "count": 29, "max_ms": 10164.059, "p95_ms": 10115.887}, "triton_wait": {"avg_ms": 74.462, "count": 29, "max_ms": 104.679, "p95_ms": 100.227}, "visualize": {"avg_ms": 4.206, "count": 29, "max_ms": 6.505, "p95_ms": 5.184}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 811}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5505.027, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:22:39+09:00", "last_error_summary": "[NULL @ 0x55c36f79f200] missing picture in access unit with size 18 [h264 @ 0x55c36f9568c0] no frame! [in#0/rtsp @ 0x55c36f79ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c36f79ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c36f79f840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:22:39+09:00", "last_error_summary": "[NULL @ 0x55c36f79f200] missing picture in access unit with size 18 [h264 @ 0x55c36f9568c0] no frame! [in#0/rtsp @ 0x55c36f79ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c36f79ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c36f79f840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T16:22:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.03, "last_started_at": "2026-07-06T16:19:59+09:00", "pid": 16428, "process": {"exists": false, "pid": 16428}, "restart_count": 41, "restart_count_per_hour": 12, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.59, "sample_count": 1576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97418, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:22:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T16:22:39+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T16:22:39+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T16:22:39.305385+09:00", "mtime_epoch": 1783322559.305385, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254025}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:22:39+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1352.801, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.97, "pushed": 0.97, "read": 0.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.013, "count": 29, "max_ms": 9.627, "p95_ms": 8.172}, "gstreamer_push": {"avg_ms": 1.324, "count": 29, "max_ms": 3.616, "p95_ms": 1.579}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.766, "count": 29, "max_ms": 75.181, "p95_ms": 74.992}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.777, "count": 29, "max_ms": 104.667, "p95_ms": 100.215}, "yolo11_640_fire_11class": {"avg_ms": 64.199, "count": 29, "max_ms": 85.451, "p95_ms": 84.756}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.092, "count": 29, "max_ms": 102.287, "p95_ms": 97.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.48, "count": 29, "max_ms": 151.04, "p95_ms": 149.848}, "read_interval": {"avg_ms": 1067.581, "count": 28, "max_ms": 10164.059, "p95_ms": 10115.887}, "triton_wait": {"avg_ms": 73.511, "count": 29, "max_ms": 104.679, "p95_ms": 100.227}, "visualize": {"avg_ms": 4.306, "count": 29, "max_ms": 6.587, "p95_ms": 6.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 811}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5505.02, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:22:39+09:00", "last_error_summary": "[NULL @ 0x55c36f79f200] missing picture in access unit with size 18 [h264 @ 0x55c36f9568c0] no frame! [in#0/rtsp @ 0x55c36f79ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c36f79ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c36f79f840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T16:22:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.034, "last_started_at": "2026-07-06T16:19:59+09:00", "pid": 16428, "process": {"exists": false, "pid": 16428}, "restart_count": 41, "restart_count_per_hour": 12, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.59, "sample_count": 1576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97418, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:22:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T16:22:39+09:00", "last_frame_read_age_sec": 0.135, "last_frame_read_at": "2026-07-06T16:22:39+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T16:22:39.305385+09:00", "mtime_epoch": 1783322559.305385, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254025}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:22:39+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1352.801, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.969, "pushed": 0.969, "read": 0.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.013, "count": 29, "max_ms": 9.627, "p95_ms": 8.172}, "gstreamer_push": {"avg_ms": 1.324, "count": 29, "max_ms": 3.616, "p95_ms": 1.579}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.766, "count": 29, "max_ms": 75.181, "p95_ms": 74.992}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.777, "count": 29, "max_ms": 104.667, "p95_ms": 100.215}, "yolo11_640_fire_11class": {"avg_ms": 64.199, "count": 29, "max_ms": 85.451, "p95_ms": 84.756}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.092, "count": 29, "max_ms": 102.287, "p95_ms": 97.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.48, "count": 29, "max_ms": 151.04, "p95_ms": 149.848}, "read_interval": {"avg_ms": 1067.581, "count": 28, "max_ms": 10164.059, "p95_ms": 10115.887}, "triton_wait": {"avg_ms": 73.511, "count": 29, "max_ms": 104.679, "p95_ms": 100.227}, "visualize": {"avg_ms": 4.306, "count": 29, "max_ms": 6.587, "p95_ms": 6.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 811}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5550.211, "limit_mb": 8796093022207.996}, "detail": {"pid": 16687, "restart_count": 42, "start_count": 43}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:22:39+09:00", "last_error_summary": "[NULL @ 0x55c36f79f200] missing picture in access unit with size 18 [h264 @ 0x55c36f9568c0] no frame! [in#0/rtsp @ 0x55c36f79ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c36f79ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c36f79f840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T16:22:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:22:41+09:00", "pid": 16687, "process": {"exists": true, "fd_count": 5, "hwm_mb": 81.086, "name": "ffmpeg", "pid": 16687, "rss_mb": 81.086, "state": "S (sleeping)", "threads": 19}, "restart_count": 42, "restart_count_per_hour": 13, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.59, "sample_count": 1572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97418, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:22:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.369, "last_frame_pushed_at": "2026-07-06T16:22:39+09:00", "last_frame_read_age_sec": 4.493, "last_frame_read_at": "2026-07-06T16:22:39+09:00", "mjpeg": {"age_sec": 4.351, "exists": true, "mtime": "2026-07-06T16:22:39.433388+09:00", "mtime_epoch": 1783322559.4333878, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253104}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:22:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1352.801, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.542, "pushed": 0.542, "read": 0.542}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.972, "count": 14, "max_ms": 9.627, "p95_ms": 9.627}, "gstreamer_push": {"avg_ms": 1.284, "count": 14, "max_ms": 1.579, "p95_ms": 1.579}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.646, "count": 14, "max_ms": 63.778, "p95_ms": 63.778}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.063, "count": 14, "max_ms": 96.592, "p95_ms": 96.592}, "yolo11_640_fire_11class": {"avg_ms": 63.602, "count": 14, "max_ms": 84.756, "p95_ms": 84.756}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.684, "count": 14, "max_ms": 91.045, "p95_ms": 91.045}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.917, "count": 14, "max_ms": 145.693, "p95_ms": 145.693}, "read_interval": {"avg_ms": 2019.183, "count": 14, "max_ms": 10164.059, "p95_ms": 10164.059}, "triton_wait": {"avg_ms": 71.309, "count": 14, "max_ms": 96.604, "p95_ms": 96.604}, "visualize": {"avg_ms": 4.319, "count": 14, "max_ms": 6.587, "p95_ms": 6.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 25.844, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 948}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5804.266, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5601bdc1e1c0] error while decoding MB 43 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 176, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:24:00+09:00", "last_error_summary": "[h264 @ 0x5601bdc1e1c0] error while decoding MB 43 37, bytestream -5", "last_exit_at": "2026-07-06T16:20:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 188.039, "last_started_at": "2026-07-06T16:20:52+09:00", "pid": 16580, "process": {"exists": false, "pid": 16580}, "restart_count": 32, "restart_count_per_hour": 11, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 197.01, "sample_count": 2044, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 113871, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:24:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.048, "last_frame_pushed_at": "2026-07-06T16:24:00+09:00", "last_frame_read_age_sec": 0.039, "last_frame_read_at": "2026-07-06T16:24:00+09:00", "mjpeg": {"age_sec": 0.031, "exists": true, "mtime": "2026-07-06T16:24:00.839085+09:00", "mtime_epoch": 1783322640.8390849, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192819}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:24:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1718.379, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.999, "pushed": 5.999, "read": 5.999}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.608, "count": 179, "max_ms": 13.007, "p95_ms": 10.881}, "gstreamer_push": {"avg_ms": 0.982, "count": 179, "max_ms": 1.134, "p95_ms": 1.093}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.975, "count": 179, "max_ms": 90.979, "p95_ms": 63.941}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.511, "count": 179, "max_ms": 113.75, "p95_ms": 94.599}, "yolo11_640_fire_11class": {"avg_ms": 56.204, "count": 179, "max_ms": 91.516, "p95_ms": 72.127}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.532, "count": 179, "max_ms": 110.749, "p95_ms": 84.887}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.716, "count": 179, "max_ms": 267.34, "p95_ms": 228.881}, "read_interval": {"avg_ms": 167.764, "count": 179, "max_ms": 275.774, "p95_ms": 236.188}, "triton_wait": {"avg_ms": 69.787, "count": 179, "max_ms": 113.764, "p95_ms": 94.988}, "visualize": {"avg_ms": 7.398, "count": 179, "max_ms": 15.477, "p95_ms": 13.368}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.839, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 948}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5795.066, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:24:00+09:00", "last_error_summary": "[h264 @ 0x5601bdc1e1c0] error while decoding MB 43 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 176, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:24:00+09:00", "last_error_summary": "[h264 @ 0x5601bdc1e1c0] error while decoding MB 43 37, bytestream -5", "last_exit_at": "2026-07-06T16:24:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 188.146, "last_started_at": "2026-07-06T16:20:52+09:00", "pid": 16580, "process": {"exists": false, "pid": 16580}, "restart_count": 32, "restart_count_per_hour": 11, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 197.01, "sample_count": 2044, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 113872, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:24:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:24:00+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-06T16:24:00+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-06T16:24:00.839085+09:00", "mtime_epoch": 1783322640.8390849, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192819}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:24:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1718.379, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.011, "pushed": 6.011, "read": 5.977}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.585, "count": 179, "max_ms": 13.007, "p95_ms": 10.881}, "gstreamer_push": {"avg_ms": 0.983, "count": 180, "max_ms": 1.134, "p95_ms": 1.087}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.001, "count": 179, "max_ms": 90.979, "p95_ms": 63.941}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.405, "count": 179, "max_ms": 113.75, "p95_ms": 94.599}, "yolo11_640_fire_11class": {"avg_ms": 56.217, "count": 179, "max_ms": 91.516, "p95_ms": 72.127}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.533, "count": 179, "max_ms": 110.749, "p95_ms": 84.887}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.61, "count": 180, "max_ms": 267.34, "p95_ms": 228.654}, "read_interval": {"avg_ms": 167.764, "count": 179, "max_ms": 275.774, "p95_ms": 236.188}, "triton_wait": {"avg_ms": 69.685, "count": 179, "max_ms": 113.764, "p95_ms": 94.988}, "visualize": {"avg_ms": 7.35, "count": 179, "max_ms": 15.477, "p95_ms": 13.317}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 948}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5795.184, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 176, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:24:00+09:00", "last_error_summary": "[h264 @ 0x5601bdc1e1c0] error while decoding MB 43 37, bytestream -5", "last_exit_at": "2026-07-06T16:24:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 188.151, "last_started_at": "2026-07-06T16:20:52+09:00", "pid": 16580, "process": {"exists": false, "pid": 16580}, "restart_count": 32, "restart_count_per_hour": 11, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 197.01, "sample_count": 2044, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 113872, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:24:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T16:24:00+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T16:24:00+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T16:24:00.839085+09:00", "mtime_epoch": 1783322640.8390849, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192819}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:24:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1718.379, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.01, "pushed": 6.01, "read": 5.977}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.585, "count": 179, "max_ms": 13.007, "p95_ms": 10.881}, "gstreamer_push": {"avg_ms": 0.983, "count": 180, "max_ms": 1.134, "p95_ms": 1.087}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.001, "count": 179, "max_ms": 90.979, "p95_ms": 63.941}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.405, "count": 179, "max_ms": 113.75, "p95_ms": 94.599}, "yolo11_640_fire_11class": {"avg_ms": 56.217, "count": 179, "max_ms": 91.516, "p95_ms": 72.127}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.533, "count": 179, "max_ms": 110.749, "p95_ms": 84.887}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.61, "count": 180, "max_ms": 267.34, "p95_ms": 228.654}, "read_interval": {"avg_ms": 167.764, "count": 179, "max_ms": 275.774, "p95_ms": 236.188}, "triton_wait": {"avg_ms": 69.685, "count": 179, "max_ms": 113.764, "p95_ms": 94.988}, "visualize": {"avg_ms": 7.35, "count": 179, "max_ms": 15.477, "p95_ms": 13.317}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 948}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5905.566, "limit_mb": 8796093022207.996}, "detail": {"pid": 16899, "restart_count": 33, "start_count": 34}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 176, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:24:00+09:00", "last_error_summary": "[h264 @ 0x5601bdc1e1c0] error while decoding MB 43 37, bytestream -5", "last_exit_at": "2026-07-06T16:24:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:24:03+09:00", "pid": 16899, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.992, "name": "ffmpeg", "pid": 16899, "rss_mb": 138.992, "state": "S (sleeping)", "threads": 106}, "restart_count": 33, "restart_count_per_hour": 12, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 197.322, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 113872, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:24:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.404, "last_frame_pushed_at": "2026-07-06T16:24:00+09:00", "last_frame_read_age_sec": 4.546, "last_frame_read_at": "2026-07-06T16:24:00+09:00", "mjpeg": {"age_sec": 4.386, "exists": true, "mtime": "2026-07-06T16:24:00.991088+09:00", "mtime_epoch": 1783322640.991088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193203}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:24:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1718.379, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.347, "pushed": 5.347, "read": 5.313}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.231, "count": 159, "max_ms": 13.007, "p95_ms": 9.831}, "gstreamer_push": {"avg_ms": 0.996, "count": 160, "max_ms": 1.134, "p95_ms": 1.093}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.937, "count": 159, "max_ms": 90.979, "p95_ms": 63.941}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.523, "count": 159, "max_ms": 113.75, "p95_ms": 93.803}, "yolo11_640_fire_11class": {"avg_ms": 55.928, "count": 159, "max_ms": 91.516, "p95_ms": 72.12}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.939, "count": 159, "max_ms": 110.749, "p95_ms": 82.81}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.804, "count": 160, "max_ms": 267.34, "p95_ms": 198.125}, "read_interval": {"avg_ms": 161.01, "count": 159, "max_ms": 275.774, "p95_ms": 206.03}, "triton_wait": {"avg_ms": 68.822, "count": 159, "max_ms": 113.764, "p95_ms": 94.659}, "visualize": {"avg_ms": 6.656, "count": 159, "max_ms": 15.477, "p95_ms": 12.792}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 815}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5736.109, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55df63697540] error while decoding MB 78 7, bytestream -16 [in#0/rtsp @ 0x55df6367be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55df6367be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:24:12+09:00", "last_error_summary": "[h264 @ 0x55df63697540] error while decoding MB 78 7, bytestream -16 [in#0/rtsp @ 0x55df6367be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55df6367be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:22:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 90.821, "last_started_at": "2026-07-06T16:22:41+09:00", "pid": 16687, "process": {"exists": false, "pid": 16687}, "restart_count": 42, "restart_count_per_hour": 13, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.762, "sample_count": 1603, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97886, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:24:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-06T16:24:12+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T16:24:12+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-06T16:24:12.559327+09:00", "mtime_epoch": 1783322652.5593274, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:24:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1353.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.69, "pushed": 0.69, "read": 0.747}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.707, "count": 12, "max_ms": 6.862, "p95_ms": 6.862}, "gstreamer_push": {"avg_ms": 1.138, "count": 12, "max_ms": 1.442, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.454, "count": 12, "max_ms": 69.292, "p95_ms": 69.292}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.575, "count": 12, "max_ms": 112.432, "p95_ms": 112.432}, "yolo11_640_fire_11class": {"avg_ms": 60.709, "count": 12, "max_ms": 74.422, "p95_ms": 74.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.334, "count": 12, "max_ms": 88.907, "p95_ms": 88.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.876, "count": 12, "max_ms": 136.294, "p95_ms": 136.294}, "read_interval": {"avg_ms": 2447.31, "count": 13, "max_ms": 15987.218, "p95_ms": 15987.218}, "triton_wait": {"avg_ms": 69.161, "count": 12, "max_ms": 112.446, "p95_ms": 112.446}, "visualize": {"avg_ms": 4.773, "count": 12, "max_ms": 5.766, "p95_ms": 5.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.402, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 815}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5735.242, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:24:12+09:00", "last_error_summary": "[h264 @ 0x55df63697540] error while decoding MB 78 7, bytestream -16 [in#0/rtsp @ 0x55df6367be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55df6367be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:24:12+09:00", "last_error_summary": "[h264 @ 0x55df63697540] error while decoding MB 78 7, bytestream -16 [in#0/rtsp @ 0x55df6367be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55df6367be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:24:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 90.892, "last_started_at": "2026-07-06T16:22:41+09:00", "pid": 16687, "process": {"exists": false, "pid": 16687}, "restart_count": 42, "restart_count_per_hour": 13, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.762, "sample_count": 1604, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:24:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T16:24:12+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-06T16:24:12+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T16:24:12.559327+09:00", "mtime_epoch": 1783322652.5593274, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:24:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1353.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.848, "count": 13, "max_ms": 7.532, "p95_ms": 7.532}, "gstreamer_push": {"avg_ms": 1.143, "count": 13, "max_ms": 1.442, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.438, "count": 13, "max_ms": 69.292, "p95_ms": 69.292}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.472, "count": 13, "max_ms": 112.432, "p95_ms": 112.432}, "yolo11_640_fire_11class": {"avg_ms": 60.193, "count": 13, "max_ms": 74.422, "p95_ms": 74.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.876, "count": 13, "max_ms": 88.907, "p95_ms": 88.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.767, "count": 13, "max_ms": 136.294, "p95_ms": 136.294}, "read_interval": {"avg_ms": 2447.31, "count": 13, "max_ms": 15987.218, "p95_ms": 15987.218}, "triton_wait": {"avg_ms": 69.015, "count": 13, "max_ms": 112.446, "p95_ms": 112.446}, "visualize": {"avg_ms": 4.786, "count": 13, "max_ms": 5.766, "p95_ms": 5.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.472, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 815}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5735.164, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:24:12+09:00", "last_error_summary": "[h264 @ 0x55df63697540] error while decoding MB 78 7, bytestream -16 [in#0/rtsp @ 0x55df6367be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55df6367be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:24:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 90.896, "last_started_at": "2026-07-06T16:22:41+09:00", "pid": 16687, "process": {"exists": false, "pid": 16687}, "restart_count": 42, "restart_count_per_hour": 13, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.762, "sample_count": 1604, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:24:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T16:24:12+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T16:24:12+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T16:24:12.559327+09:00", "mtime_epoch": 1783322652.5593274, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:24:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1353.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.848, "count": 13, "max_ms": 7.532, "p95_ms": 7.532}, "gstreamer_push": {"avg_ms": 1.143, "count": 13, "max_ms": 1.442, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.438, "count": 13, "max_ms": 69.292, "p95_ms": 69.292}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.472, "count": 13, "max_ms": 112.432, "p95_ms": 112.432}, "yolo11_640_fire_11class": {"avg_ms": 60.193, "count": 13, "max_ms": 74.422, "p95_ms": 74.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.876, "count": 13, "max_ms": 88.907, "p95_ms": 88.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.767, "count": 13, "max_ms": 136.294, "p95_ms": 136.294}, "read_interval": {"avg_ms": 2447.31, "count": 13, "max_ms": 15987.218, "p95_ms": 15987.218}, "triton_wait": {"avg_ms": 69.015, "count": 13, "max_ms": 112.446, "p95_ms": 112.446}, "visualize": {"avg_ms": 4.786, "count": 13, "max_ms": 5.766, "p95_ms": 5.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.476, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 815}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5904.961, "limit_mb": 8796093022207.996}, "detail": {"pid": 17006, "restart_count": 43, "start_count": 44}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:24:12+09:00", "last_error_summary": "[h264 @ 0x55df63697540] error while decoding MB 78 7, bytestream -16 [in#0/rtsp @ 0x55df6367be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55df6367be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:24:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:24:15+09:00", "pid": 17006, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.652, "name": "ffmpeg", "pid": 17006, "rss_mb": 197.652, "state": "S (sleeping)", "threads": 106}, "restart_count": 43, "restart_count_per_hour": 14, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.676, "sample_count": 1574, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:24:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.532, "last_frame_pushed_at": "2026-07-06T16:24:12+09:00", "last_frame_read_age_sec": 4.658, "last_frame_read_at": "2026-07-06T16:24:12+09:00", "mjpeg": {"age_sec": 4.515, "exists": true, "mtime": "2026-07-06T16:24:12.687330+09:00", "mtime_epoch": 1783322652.68733, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233689}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:24:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1353.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.591, "pushed": 0.591, "read": 0.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.848, "count": 13, "max_ms": 7.532, "p95_ms": 7.532}, "gstreamer_push": {"avg_ms": 1.143, "count": 13, "max_ms": 1.442, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.438, "count": 13, "max_ms": 69.292, "p95_ms": 69.292}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.472, "count": 13, "max_ms": 112.432, "p95_ms": 112.432}, "yolo11_640_fire_11class": {"avg_ms": 60.193, "count": 13, "max_ms": 74.422, "p95_ms": 74.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.876, "count": 13, "max_ms": 88.907, "p95_ms": 88.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.767, "count": 13, "max_ms": 136.294, "p95_ms": 136.294}, "read_interval": {"avg_ms": 2447.31, "count": 13, "max_ms": 15987.218, "p95_ms": 15987.218}, "triton_wait": {"avg_ms": 69.015, "count": 13, "max_ms": 112.446, "p95_ms": 112.446}, "visualize": {"avg_ms": 4.786, "count": 13, "max_ms": 5.766, "p95_ms": 5.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 971}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5814.715, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562ad02fd140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad02fd140] error while decoding MB 0 3, bytestream 7654 [h264 @ 0x562ad0311240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0311240] error while decoding MB 0 7, bytestream 3198 [h264 @ 0x562ad0350700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0350700] error while decoding MB 0 31, bytestream 4250 [h264 @ 0x562ad02e8240] cabac decode of qscale diff failed", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:29:27+09:00", "last_error_summary": "[h264 @ 0x562ad02fd140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad02fd140] error while decoding MB 0 3, bytestream 7654 [h264 @ 0x562ad0311240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0311240] error while decoding MB 0 7, bytestream 3198 [h264 @ 0x562ad0350700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0350700] error while decoding MB 0 31, bytestream 4250 [h264 @ 0x562ad02e8240] cabac decode of qscale diff failed", "last_exit_at": "2026-07-06T16:24:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 324.365, "last_started_at": "2026-07-06T16:24:03+09:00", "pid": 16899, "process": {"exists": false, "pid": 16899}, "restart_count": 33, "restart_count_per_hour": 12, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.129, "sample_count": 2534, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 116615, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:29:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T16:29:27+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T16:29:27+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T16:29:27.697723+09:00", "mtime_epoch": 1783322967.6977232, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192052}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:29:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1719.664, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 108, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.605, "pushed": 3.605, "read": 3.605}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.615, "count": 108, "max_ms": 5.628, "p95_ms": 5.362}, "gstreamer_push": {"avg_ms": 1.29, "count": 108, "max_ms": 1.838, "p95_ms": 1.456}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.393, "count": 108, "max_ms": 87.716, "p95_ms": 75.833}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.84, "count": 108, "max_ms": 111.695, "p95_ms": 102.15}, "yolo11_640_fire_11class": {"avg_ms": 59.89, "count": 108, "max_ms": 90.375, "p95_ms": 81.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.431, "count": 108, "max_ms": 107.281, "p95_ms": 98.582}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.21, "count": 108, "max_ms": 145.995, "p95_ms": 119.634}, "read_interval": {"avg_ms": 277.534, "count": 108, "max_ms": 10125.07, "p95_ms": 138.685}, "triton_wait": {"avg_ms": 73.259, "count": 108, "max_ms": 111.708, "p95_ms": 102.699}, "visualize": {"avg_ms": 3.692, "count": 108, "max_ms": 5.237, "p95_ms": 4.612}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 971}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5814.223, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:29:27+09:00", "last_error_summary": "[h264 @ 0x562ad02fd140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad02fd140] error while decoding MB 0 3, bytestream 7654 [h264 @ 0x562ad0311240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0311240] error while decoding MB 0 7, bytestream 3198 [h264 @ 0x562ad0350700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0350700] error while decoding MB 0 31, bytestream 4250 [h264 @ 0x562ad02e8240] cabac decode of qscale diff failed", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:29:27+09:00", "last_error_summary": "[h264 @ 0x562ad02fd140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad02fd140] error while decoding MB 0 3, bytestream 7654 [h264 @ 0x562ad0311240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0311240] error while decoding MB 0 7, bytestream 3198 [h264 @ 0x562ad0350700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0350700] error while decoding MB 0 31, bytestream 4250 [h264 @ 0x562ad02e8240] cabac decode of qscale diff failed", "last_exit_at": "2026-07-06T16:29:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 324.404, "last_started_at": "2026-07-06T16:24:03+09:00", "pid": 16899, "process": {"exists": false, "pid": 16899}, "restart_count": 33, "restart_count_per_hour": 12, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.129, "sample_count": 2535, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 116616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:29:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T16:29:27+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-06T16:29:27+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-06T16:29:27.697723+09:00", "mtime_epoch": 1783322967.6977232, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192052}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:29:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1719.664, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.634, "pushed": 3.634, "read": 3.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.647, "count": 109, "max_ms": 7.131, "p95_ms": 5.454}, "gstreamer_push": {"avg_ms": 1.291, "count": 109, "max_ms": 1.838, "p95_ms": 1.456}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.302, "count": 109, "max_ms": 87.716, "p95_ms": 75.833}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.717, "count": 109, "max_ms": 111.695, "p95_ms": 102.15}, "yolo11_640_fire_11class": {"avg_ms": 59.81, "count": 109, "max_ms": 90.375, "p95_ms": 81.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.337, "count": 109, "max_ms": 107.281, "p95_ms": 98.582}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.1, "count": 109, "max_ms": 145.995, "p95_ms": 119.634}, "read_interval": {"avg_ms": 277.534, "count": 108, "max_ms": 10125.07, "p95_ms": 138.685}, "triton_wait": {"avg_ms": 73.132, "count": 109, "max_ms": 111.708, "p95_ms": 102.699}, "visualize": {"avg_ms": 3.693, "count": 109, "max_ms": 5.237, "p95_ms": 4.612}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 971}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5814.102, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:29:27+09:00", "last_error_summary": "[h264 @ 0x562ad02fd140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad02fd140] error while decoding MB 0 3, bytestream 7654 [h264 @ 0x562ad0311240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0311240] error while decoding MB 0 7, bytestream 3198 [h264 @ 0x562ad0350700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0350700] error while decoding MB 0 31, bytestream 4250 [h264 @ 0x562ad02e8240] cabac decode of qscale diff failed", "last_exit_at": "2026-07-06T16:29:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 324.414, "last_started_at": "2026-07-06T16:24:03+09:00", "pid": 16899, "process": {"exists": false, "pid": 16899}, "restart_count": 33, "restart_count_per_hour": 12, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.129, "sample_count": 2535, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 116616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:29:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.025, "last_frame_pushed_at": "2026-07-06T16:29:27+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-06T16:29:27+09:00", "mjpeg": {"age_sec": 0.008, "exists": true, "mtime": "2026-07-06T16:29:27.781725+09:00", "mtime_epoch": 1783322967.7817247, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:29:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1719.664, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 108, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.601, "pushed": 3.601, "read": 3.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.65, "count": 108, "max_ms": 7.131, "p95_ms": 5.454}, "gstreamer_push": {"avg_ms": 1.29, "count": 108, "max_ms": 1.838, "p95_ms": 1.456}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.418, "count": 108, "max_ms": 87.716, "p95_ms": 75.833}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.855, "count": 108, "max_ms": 111.695, "p95_ms": 102.15}, "yolo11_640_fire_11class": {"avg_ms": 59.92, "count": 108, "max_ms": 90.375, "p95_ms": 81.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.467, "count": 108, "max_ms": 107.281, "p95_ms": 98.582}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.234, "count": 108, "max_ms": 145.995, "p95_ms": 119.634}, "read_interval": {"avg_ms": 277.534, "count": 108, "max_ms": 10125.07, "p95_ms": 138.685}, "triton_wait": {"avg_ms": 73.275, "count": 108, "max_ms": 111.708, "p95_ms": 102.699}, "visualize": {"avg_ms": 3.685, "count": 108, "max_ms": 5.237, "p95_ms": 4.548}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 971}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5921.383, "limit_mb": 8796093022207.996}, "detail": {"pid": 17158, "restart_count": 34, "start_count": 35}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:29:27+09:00", "last_error_summary": "[h264 @ 0x562ad02fd140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad02fd140] error while decoding MB 0 3, bytestream 7654 [h264 @ 0x562ad0311240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0311240] error while decoding MB 0 7, bytestream 3198 [h264 @ 0x562ad0350700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562ad0350700] error while decoding MB 0 31, bytestream 4250 [h264 @ 0x562ad02e8240] cabac decode of qscale diff failed", "last_exit_at": "2026-07-06T16:29:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:29:30+09:00", "pid": 17158, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.406, "name": "ffmpeg", "pid": 17158, "rss_mb": 141.406, "state": "S (sleeping)", "threads": 106}, "restart_count": 34, "restart_count_per_hour": 13, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.182, "sample_count": 2494, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 116616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:29:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.26, "last_frame_pushed_at": "2026-07-06T16:29:27+09:00", "last_frame_read_age_sec": 4.337, "last_frame_read_at": "2026-07-06T16:29:27+09:00", "mjpeg": {"age_sec": 4.243, "exists": true, "mtime": "2026-07-06T16:29:27.781725+09:00", "mtime_epoch": 1783322967.7817247, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:29:32+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1719.664, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 65, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.205, "pushed": 2.205, "read": 2.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.745, "count": 66, "max_ms": 7.131, "p95_ms": 5.454}, "gstreamer_push": {"avg_ms": 1.265, "count": 66, "max_ms": 1.523, "p95_ms": 1.412}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.86, "count": 66, "max_ms": 87.716, "p95_ms": 73.333}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.314, "count": 66, "max_ms": 111.695, "p95_ms": 102.15}, "yolo11_640_fire_11class": {"avg_ms": 60.627, "count": 66, "max_ms": 90.375, "p95_ms": 84.081}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.79, "count": 66, "max_ms": 106.655, "p95_ms": 98.582}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.006, "count": 66, "max_ms": 145.995, "p95_ms": 129.539}, "read_interval": {"avg_ms": 395.448, "count": 65, "max_ms": 10125.07, "p95_ms": 151.291}, "triton_wait": {"avg_ms": 72.92, "count": 66, "max_ms": 111.708, "p95_ms": 102.699}, "visualize": {"avg_ms": 3.722, "count": 66, "max_ms": 5.237, "p95_ms": 5.073}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 983}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5797.691, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556a9994d980] error while decoding MB 14 36, bytestream -21", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 183, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:32:45+09:00", "last_error_summary": "[h264 @ 0x556a9994d980] error while decoding MB 14 36, bytestream -21", "last_exit_at": "2026-07-06T16:29:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.711, "last_started_at": "2026-07-06T16:29:30+09:00", "pid": 17158, "process": {"exists": false, "pid": 17158}, "restart_count": 34, "restart_count_per_hour": 12, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.709, "sample_count": 2152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 118032, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:32:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-06T16:32:45+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T16:32:45+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-06T16:32:45.685651+09:00", "mtime_epoch": 1783323165.6856513, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198701}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:32:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1702.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.856, "pushed": 5.856, "read": 5.856}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.793, "count": 175, "max_ms": 11.261, "p95_ms": 9.475}, "gstreamer_push": {"avg_ms": 0.994, "count": 175, "max_ms": 1.556, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.814, "count": 175, "max_ms": 82.844, "p95_ms": 76.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.097, "count": 175, "max_ms": 114.473, "p95_ms": 103.401}, "yolo11_640_fire_11class": {"avg_ms": 59.213, "count": 175, "max_ms": 92.026, "p95_ms": 81.359}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.032, "count": 175, "max_ms": 104.591, "p95_ms": 96.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.342, "count": 175, "max_ms": 237.43, "p95_ms": 207.481}, "read_interval": {"avg_ms": 171.464, "count": 175, "max_ms": 244.055, "p95_ms": 216.039}, "triton_wait": {"avg_ms": 74.37, "count": 175, "max_ms": 114.487, "p95_ms": 104.271}, "visualize": {"avg_ms": 6.89, "count": 175, "max_ms": 9.708, "p95_ms": 8.692}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 983}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5797.051, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:32:45+09:00", "last_error_summary": "[h264 @ 0x556a9994d980] error while decoding MB 14 36, bytestream -21", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 183, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:32:45+09:00", "last_error_summary": "[h264 @ 0x556a9994d980] error while decoding MB 14 36, bytestream -21", "last_exit_at": "2026-07-06T16:32:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.86, "last_started_at": "2026-07-06T16:29:30+09:00", "pid": 17158, "process": {"exists": false, "pid": 17158}, "restart_count": 34, "restart_count_per_hour": 12, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.72, "sample_count": 2152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 118033, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:32:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:32:45+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-06T16:32:45+09:00", "mjpeg": {"age_sec": 0.198, "exists": true, "mtime": "2026-07-06T16:32:45.685651+09:00", "mtime_epoch": 1783323165.6856513, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198701}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:32:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1702.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.859, "pushed": 5.859, "read": 5.826}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.792, "count": 175, "max_ms": 11.261, "p95_ms": 9.475}, "gstreamer_push": {"avg_ms": 0.993, "count": 175, "max_ms": 1.556, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.998, "count": 175, "max_ms": 82.844, "p95_ms": 76.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.215, "count": 175, "max_ms": 114.473, "p95_ms": 103.401}, "yolo11_640_fire_11class": {"avg_ms": 59.326, "count": 175, "max_ms": 92.026, "p95_ms": 81.359}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.152, "count": 175, "max_ms": 104.591, "p95_ms": 96.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.458, "count": 175, "max_ms": 237.43, "p95_ms": 207.481}, "read_interval": {"avg_ms": 171.352, "count": 174, "max_ms": 244.055, "p95_ms": 216.039}, "triton_wait": {"avg_ms": 74.488, "count": 175, "max_ms": 114.487, "p95_ms": 104.271}, "visualize": {"avg_ms": 6.893, "count": 175, "max_ms": 9.708, "p95_ms": 8.692}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 983}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5797.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 183, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:32:45+09:00", "last_error_summary": "[h264 @ 0x556a9994d980] error while decoding MB 14 36, bytestream -21", "last_exit_at": "2026-07-06T16:32:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.865, "last_started_at": "2026-07-06T16:29:30+09:00", "pid": 17158, "process": {"exists": false, "pid": 17158}, "restart_count": 34, "restart_count_per_hour": 12, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.72, "sample_count": 2152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 118033, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:32:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T16:32:45+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-06T16:32:45+09:00", "mjpeg": {"age_sec": 0.202, "exists": true, "mtime": "2026-07-06T16:32:45.685651+09:00", "mtime_epoch": 1783323165.6856513, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198701}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:32:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1702.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.858, "pushed": 5.858, "read": 5.825}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.792, "count": 175, "max_ms": 11.261, "p95_ms": 9.475}, "gstreamer_push": {"avg_ms": 0.993, "count": 175, "max_ms": 1.556, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.998, "count": 175, "max_ms": 82.844, "p95_ms": 76.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.215, "count": 175, "max_ms": 114.473, "p95_ms": 103.401}, "yolo11_640_fire_11class": {"avg_ms": 59.326, "count": 175, "max_ms": 92.026, "p95_ms": 81.359}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.152, "count": 175, "max_ms": 104.591, "p95_ms": 96.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.458, "count": 175, "max_ms": 237.43, "p95_ms": 207.481}, "read_interval": {"avg_ms": 171.352, "count": 174, "max_ms": 244.055, "p95_ms": 216.039}, "triton_wait": {"avg_ms": 74.488, "count": 175, "max_ms": 114.487, "p95_ms": 104.271}, "visualize": {"avg_ms": 6.893, "count": 175, "max_ms": 9.708, "p95_ms": 8.692}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 983}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5904.039, "limit_mb": 8796093022207.996}, "detail": {"pid": 17265, "restart_count": 35, "start_count": 36}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 183, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:32:45+09:00", "last_error_summary": "[h264 @ 0x556a9994d980] error while decoding MB 14 36, bytestream -21", "last_exit_at": "2026-07-06T16:32:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:32:48+09:00", "pid": 17265, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.746, "name": "ffmpeg", "pid": 17265, "rss_mb": 136.746, "state": "S (sleeping)", "threads": 106}, "restart_count": 35, "restart_count_per_hour": 13, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 184.1, "sample_count": 2116, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 118033, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:32:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.639, "last_frame_pushed_at": "2026-07-06T16:32:45+09:00", "last_frame_read_age_sec": 4.845, "last_frame_read_at": "2026-07-06T16:32:45+09:00", "mjpeg": {"age_sec": 4.621, "exists": true, "mtime": "2026-07-06T16:32:45.897655+09:00", "mtime_epoch": 1783323165.8976555, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195227}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:32:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1702.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.077, "pushed": 5.077, "read": 5.043}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.637, "count": 151, "max_ms": 11.261, "p95_ms": 9.242}, "gstreamer_push": {"avg_ms": 1.006, "count": 152, "max_ms": 1.556, "p95_ms": 1.202}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.16, "count": 151, "max_ms": 82.844, "p95_ms": 76.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.797, "count": 151, "max_ms": 111.824, "p95_ms": 103.249}, "yolo11_640_fire_11class": {"avg_ms": 59.32, "count": 151, "max_ms": 92.026, "p95_ms": 80.739}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.895, "count": 151, "max_ms": 104.591, "p95_ms": 96.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.445, "count": 152, "max_ms": 218.535, "p95_ms": 201.467}, "read_interval": {"avg_ms": 167.413, "count": 151, "max_ms": 225.737, "p95_ms": 208.058}, "triton_wait": {"avg_ms": 74.112, "count": 151, "max_ms": 111.836, "p95_ms": 103.26}, "visualize": {"avg_ms": 6.7, "count": 151, "max_ms": 9.708, "p95_ms": 8.213}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5687.391, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55eb08d97340] error while decoding MB 4 42, bytestream -27", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 184, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:34:49+09:00", "last_error_summary": "[h264 @ 0x55eb08d97340] error while decoding MB 4 42, bytestream -27", "last_exit_at": "2026-07-06T16:32:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.016, "last_started_at": "2026-07-06T16:32:48+09:00", "pid": 17265, "process": {"exists": false, "pid": 17265}, "restart_count": 35, "restart_count_per_hour": 13, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.156, "sample_count": 1942, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 118683, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:34:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.045, "last_frame_pushed_at": "2026-07-06T16:34:49+09:00", "last_frame_read_age_sec": 0.038, "last_frame_read_at": "2026-07-06T16:34:49+09:00", "mjpeg": {"age_sec": 0.022, "exists": true, "mtime": "2026-07-06T16:34:49.512085+09:00", "mtime_epoch": 1783323289.512085, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203304}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:34:49+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1599.527, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.703, "pushed": 6.703, "read": 6.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.076, "count": 200, "max_ms": 9.065, "p95_ms": 7.678}, "gstreamer_push": {"avg_ms": 1.107, "count": 201, "max_ms": 1.409, "p95_ms": 1.252}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.964, "count": 200, "max_ms": 84.158, "p95_ms": 78.259}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.76, "count": 200, "max_ms": 121.908, "p95_ms": 107.671}, "yolo11_640_fire_11class": {"avg_ms": 72.618, "count": 200, "max_ms": 94.069, "p95_ms": 86.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.179, "count": 200, "max_ms": 106.608, "p95_ms": 101.077}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.228, "count": 201, "max_ms": 189.689, "p95_ms": 163.492}, "read_interval": {"avg_ms": 149.833, "count": 201, "max_ms": 200.422, "p95_ms": 173.462}, "triton_wait": {"avg_ms": 88.522, "count": 200, "max_ms": 121.921, "p95_ms": 107.682}, "visualize": {"avg_ms": 5.122, "count": 200, "max_ms": 6.406, "p95_ms": 5.901}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5686.441, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:34:49+09:00", "last_error_summary": "[h264 @ 0x55eb08d97340] error while decoding MB 4 42, bytestream -27", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 184, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:34:49+09:00", "last_error_summary": "[h264 @ 0x55eb08d97340] error while decoding MB 4 42, bytestream -27", "last_exit_at": "2026-07-06T16:34:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.153, "last_started_at": "2026-07-06T16:32:48+09:00", "pid": 17265, "process": {"exists": false, "pid": 17265}, "restart_count": 35, "restart_count_per_hour": 13, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.156, "sample_count": 1942, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 118684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:34:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:34:49+09:00", "last_frame_read_age_sec": 0.174, "last_frame_read_at": "2026-07-06T16:34:49+09:00", "mjpeg": {"age_sec": 0.159, "exists": true, "mtime": "2026-07-06T16:34:49.512085+09:00", "mtime_epoch": 1783323289.512085, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203304}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:34:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1599.527, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.706, "pushed": 6.706, "read": 6.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.077, "count": 200, "max_ms": 9.065, "p95_ms": 7.678}, "gstreamer_push": {"avg_ms": 1.107, "count": 201, "max_ms": 1.409, "p95_ms": 1.252}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.078, "count": 200, "max_ms": 84.158, "p95_ms": 78.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.904, "count": 200, "max_ms": 121.908, "p95_ms": 110.474}, "yolo11_640_fire_11class": {"avg_ms": 72.704, "count": 200, "max_ms": 94.069, "p95_ms": 86.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.224, "count": 200, "max_ms": 106.608, "p95_ms": 101.077}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.22, "count": 201, "max_ms": 189.689, "p95_ms": 163.492}, "read_interval": {"avg_ms": 149.689, "count": 200, "max_ms": 200.422, "p95_ms": 173.074}, "triton_wait": {"avg_ms": 88.665, "count": 200, "max_ms": 121.921, "p95_ms": 110.487}, "visualize": {"avg_ms": 5.122, "count": 200, "max_ms": 6.406, "p95_ms": 5.901}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5686.281, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 184, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:34:49+09:00", "last_error_summary": "[h264 @ 0x55eb08d97340] error while decoding MB 4 42, bytestream -27", "last_exit_at": "2026-07-06T16:34:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.158, "last_started_at": "2026-07-06T16:32:48+09:00", "pid": 17265, "process": {"exists": false, "pid": 17265}, "restart_count": 35, "restart_count_per_hour": 13, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.156, "sample_count": 1942, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 118684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:34:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T16:34:49+09:00", "last_frame_read_age_sec": 0.18, "last_frame_read_at": "2026-07-06T16:34:49+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-06T16:34:49.512085+09:00", "mtime_epoch": 1783323289.512085, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203304}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:34:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1599.527, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.705, "pushed": 6.705, "read": 6.671}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.077, "count": 200, "max_ms": 9.065, "p95_ms": 7.678}, "gstreamer_push": {"avg_ms": 1.107, "count": 201, "max_ms": 1.409, "p95_ms": 1.252}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.078, "count": 200, "max_ms": 84.158, "p95_ms": 78.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.904, "count": 200, "max_ms": 121.908, "p95_ms": 110.474}, "yolo11_640_fire_11class": {"avg_ms": 72.704, "count": 200, "max_ms": 94.069, "p95_ms": 86.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.224, "count": 200, "max_ms": 106.608, "p95_ms": 101.077}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.22, "count": 201, "max_ms": 189.689, "p95_ms": 163.492}, "read_interval": {"avg_ms": 149.689, "count": 200, "max_ms": 200.422, "p95_ms": 173.074}, "triton_wait": {"avg_ms": 88.665, "count": 200, "max_ms": 121.921, "p95_ms": 110.487}, "visualize": {"avg_ms": 5.122, "count": 200, "max_ms": 6.406, "p95_ms": 5.901}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5801.047, "limit_mb": 8796093022207.996}, "detail": {"pid": 17372, "restart_count": 36, "start_count": 37}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 184, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:34:49+09:00", "last_error_summary": "[h264 @ 0x55eb08d97340] error while decoding MB 4 42, bytestream -27", "last_exit_at": "2026-07-06T16:34:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:34:52+09:00", "pid": 17372, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.824, "name": "ffmpeg", "pid": 17372, "rss_mb": 138.824, "state": "S (sleeping)", "threads": 106}, "restart_count": 36, "restart_count_per_hour": 14, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.401, "sample_count": 1912, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 118684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:34:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.556, "last_frame_pushed_at": "2026-07-06T16:34:49+09:00", "last_frame_read_age_sec": 4.725, "last_frame_read_at": "2026-07-06T16:34:49+09:00", "mjpeg": {"age_sec": 4.542, "exists": true, "mtime": "2026-07-06T16:34:49.680088+09:00", "mtime_epoch": 1783323289.6800883, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203419}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:34:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1599.527, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 170, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.705, "pushed": 5.705, "read": 5.672}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.026, "count": 170, "max_ms": 8.223, "p95_ms": 7.702}, "gstreamer_push": {"avg_ms": 1.112, "count": 171, "max_ms": 1.409, "p95_ms": 1.271}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.345, "count": 170, "max_ms": 84.158, "p95_ms": 79.248}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.763, "count": 170, "max_ms": 121.908, "p95_ms": 111.565}, "yolo11_640_fire_11class": {"avg_ms": 72.873, "count": 170, "max_ms": 94.069, "p95_ms": 87.182}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.151, "count": 170, "max_ms": 106.608, "p95_ms": 101.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.813, "count": 171, "max_ms": 189.689, "p95_ms": 164.033}, "read_interval": {"avg_ms": 149.304, "count": 170, "max_ms": 200.422, "p95_ms": 173.844}, "triton_wait": {"avg_ms": 88.556, "count": 170, "max_ms": 121.921, "p95_ms": 111.581}, "visualize": {"avg_ms": 5.129, "count": 170, "max_ms": 6.406, "p95_ms": 5.979}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 994}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5594.848, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d1c3f9e4c0] error while decoding MB 76 34, bytestream -38", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 185, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:36:57+09:00", "last_error_summary": "[h264 @ 0x55d1c3f9e4c0] error while decoding MB 76 34, bytestream -38", "last_exit_at": "2026-07-06T16:34:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.004, "last_started_at": "2026-07-06T16:34:52+09:00", "pid": 17372, "process": {"exists": false, "pid": 17372}, "restart_count": 36, "restart_count_per_hour": 14, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.12, "sample_count": 1577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 119328, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:36:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T16:36:57+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T16:36:57+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T16:36:57.186581+09:00", "mtime_epoch": 1783323417.1865807, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:36:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1497.402, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.368, "pushed": 5.368, "read": 5.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.303, "count": 160, "max_ms": 12.425, "p95_ms": 10.399}, "gstreamer_push": {"avg_ms": 1.01, "count": 160, "max_ms": 2.941, "p95_ms": 1.206}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.389, "count": 160, "max_ms": 79.515, "p95_ms": 67.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.09, "count": 160, "max_ms": 121.541, "p95_ms": 102.773}, "yolo11_640_fire_11class": {"avg_ms": 59.886, "count": 160, "max_ms": 90.705, "p95_ms": 81.917}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.359, "count": 160, "max_ms": 106.457, "p95_ms": 98.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.888, "count": 160, "max_ms": 232.314, "p95_ms": 218.696}, "read_interval": {"avg_ms": 187.342, "count": 160, "max_ms": 238.543, "p95_ms": 225.375}, "triton_wait": {"avg_ms": 76.501, "count": 160, "max_ms": 121.553, "p95_ms": 103.137}, "visualize": {"avg_ms": 7.525, "count": 160, "max_ms": 10.508, "p95_ms": 9.156}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.808, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 994}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5593.953, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:36:57+09:00", "last_error_summary": "[h264 @ 0x55d1c3f9e4c0] error while decoding MB 76 34, bytestream -38", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 185, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:36:57+09:00", "last_error_summary": "[h264 @ 0x55d1c3f9e4c0] error while decoding MB 76 34, bytestream -38", "last_exit_at": "2026-07-06T16:36:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.131, "last_started_at": "2026-07-06T16:34:52+09:00", "pid": 17372, "process": {"exists": false, "pid": 17372}, "restart_count": 36, "restart_count_per_hour": 14, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.12, "sample_count": 1577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 119329, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:36:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:36:57+09:00", "last_frame_read_age_sec": 0.178, "last_frame_read_at": "2026-07-06T16:36:57+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-06T16:36:57.186581+09:00", "mtime_epoch": 1783323417.1865807, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:36:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1497.402, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.379, "pushed": 5.379, "read": 5.345}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.331, "count": 160, "max_ms": 12.425, "p95_ms": 10.48}, "gstreamer_push": {"avg_ms": 1.01, "count": 161, "max_ms": 2.941, "p95_ms": 1.206}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.285, "count": 160, "max_ms": 79.515, "p95_ms": 67.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.871, "count": 160, "max_ms": 121.541, "p95_ms": 102.773}, "yolo11_640_fire_11class": {"avg_ms": 59.764, "count": 160, "max_ms": 90.705, "p95_ms": 81.917}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.246, "count": 160, "max_ms": 106.457, "p95_ms": 98.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.846, "count": 161, "max_ms": 232.314, "p95_ms": 218.696}, "read_interval": {"avg_ms": 187.342, "count": 160, "max_ms": 238.543, "p95_ms": 225.375}, "triton_wait": {"avg_ms": 76.283, "count": 160, "max_ms": 121.553, "p95_ms": 103.137}, "visualize": {"avg_ms": 7.539, "count": 160, "max_ms": 10.508, "p95_ms": 9.226}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 994}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5593.887, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 185, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:36:57+09:00", "last_error_summary": "[h264 @ 0x55d1c3f9e4c0] error while decoding MB 76 34, bytestream -38", "last_exit_at": "2026-07-06T16:36:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.135, "last_started_at": "2026-07-06T16:34:52+09:00", "pid": 17372, "process": {"exists": false, "pid": 17372}, "restart_count": 36, "restart_count_per_hour": 14, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.12, "sample_count": 1577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 119329, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:36:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T16:36:57+09:00", "last_frame_read_age_sec": 0.182, "last_frame_read_at": "2026-07-06T16:36:57+09:00", "mjpeg": {"age_sec": 0.169, "exists": true, "mtime": "2026-07-06T16:36:57.186581+09:00", "mtime_epoch": 1783323417.1865807, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:36:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1497.402, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.378, "pushed": 5.378, "read": 5.344}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.331, "count": 160, "max_ms": 12.425, "p95_ms": 10.48}, "gstreamer_push": {"avg_ms": 1.01, "count": 161, "max_ms": 2.941, "p95_ms": 1.206}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.285, "count": 160, "max_ms": 79.515, "p95_ms": 67.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.871, "count": 160, "max_ms": 121.541, "p95_ms": 102.773}, "yolo11_640_fire_11class": {"avg_ms": 59.764, "count": 160, "max_ms": 90.705, "p95_ms": 81.917}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.246, "count": 160, "max_ms": 106.457, "p95_ms": 98.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.846, "count": 161, "max_ms": 232.314, "p95_ms": 218.696}, "read_interval": {"avg_ms": 187.342, "count": 160, "max_ms": 238.543, "p95_ms": 225.375}, "triton_wait": {"avg_ms": 76.283, "count": 160, "max_ms": 121.553, "p95_ms": 103.137}, "visualize": {"avg_ms": 7.539, "count": 160, "max_ms": 10.508, "p95_ms": 9.226}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 994}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5705.129, "limit_mb": 8796093022207.996}, "detail": {"pid": 17479, "restart_count": 37, "start_count": 38}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 185, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:36:57+09:00", "last_error_summary": "[h264 @ 0x55d1c3f9e4c0] error while decoding MB 76 34, bytestream -38", "last_exit_at": "2026-07-06T16:37:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:37:00+09:00", "pid": 17479, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.574, "name": "ffmpeg", "pid": 17479, "rss_mb": 141.574, "state": "S (sleeping)", "threads": 106}, "restart_count": 37, "restart_count_per_hour": 15, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.428, "sample_count": 1546, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 119329, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:36:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.976, "last_frame_pushed_at": "2026-07-06T16:36:57+09:00", "last_frame_read_age_sec": 5.151, "last_frame_read_at": "2026-07-06T16:36:57+09:00", "mjpeg": {"age_sec": 4.958, "exists": true, "mtime": "2026-07-06T16:36:57.366584+09:00", "mtime_epoch": 1783323417.366584, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193436}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:37:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1497.402, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.551, "pushed": 4.551, "read": 4.518}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.332, "count": 136, "max_ms": 12.425, "p95_ms": 10.774}, "gstreamer_push": {"avg_ms": 1.004, "count": 136, "max_ms": 2.209, "p95_ms": 1.273}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.017, "count": 136, "max_ms": 79.515, "p95_ms": 66.951}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.049, "count": 136, "max_ms": 121.541, "p95_ms": 102.905}, "yolo11_640_fire_11class": {"avg_ms": 59.637, "count": 136, "max_ms": 90.705, "p95_ms": 81.339}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.082, "count": 136, "max_ms": 104.278, "p95_ms": 98.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 177.053, "count": 136, "max_ms": 229.994, "p95_ms": 218.42}, "read_interval": {"avg_ms": 184.485, "count": 135, "max_ms": 236.519, "p95_ms": 225.259}, "triton_wait": {"avg_ms": 76.182, "count": 136, "max_ms": 121.553, "p95_ms": 103.137}, "visualize": {"avg_ms": 7.458, "count": 136, "max_ms": 10.508, "p95_ms": 9.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 1000}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5595.949, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b098d1a980] error while decoding MB 49 21, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 186, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:39:11+09:00", "last_error_summary": "[h264 @ 0x55b098d1a980] error while decoding MB 49 21, bytestream -13", "last_exit_at": "2026-07-06T16:37:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.408, "last_started_at": "2026-07-06T16:37:00+09:00", "pid": 17479, "process": {"exists": false, "pid": 17479}, "restart_count": 37, "restart_count_per_hour": 15, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.592, "sample_count": 1675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 120107, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:39:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T16:39:11+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-06T16:39:11+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T16:39:11.681198+09:00", "mtime_epoch": 1783323551.6811976, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203670}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:39:11+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1492.535, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.002, "pushed": 6.002, "read": 6.002}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.122, "count": 179, "max_ms": 11.032, "p95_ms": 8.878}, "gstreamer_push": {"avg_ms": 1.09, "count": 180, "max_ms": 3.234, "p95_ms": 1.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.18, "count": 179, "max_ms": 88.05, "p95_ms": 73.101}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.771, "count": 179, "max_ms": 118.325, "p95_ms": 107.369}, "yolo11_640_fire_11class": {"avg_ms": 62.068, "count": 179, "max_ms": 88.714, "p95_ms": 82.079}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.652, "count": 179, "max_ms": 105.301, "p95_ms": 96.809}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.787, "count": 180, "max_ms": 211.313, "p95_ms": 189.687}, "read_interval": {"avg_ms": 167.199, "count": 180, "max_ms": 220.378, "p95_ms": 197.679}, "triton_wait": {"avg_ms": 79.372, "count": 179, "max_ms": 118.338, "p95_ms": 107.382}, "visualize": {"avg_ms": 6.221, "count": 179, "max_ms": 8.099, "p95_ms": 7.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 1000}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5595.156, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:39:11+09:00", "last_error_summary": "[h264 @ 0x55b098d1a980] error while decoding MB 49 21, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 186, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:39:11+09:00", "last_error_summary": "[h264 @ 0x55b098d1a980] error while decoding MB 49 21, bytestream -13", "last_exit_at": "2026-07-06T16:39:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.508, "last_started_at": "2026-07-06T16:37:00+09:00", "pid": 17479, "process": {"exists": false, "pid": 17479}, "restart_count": 37, "restart_count_per_hour": 15, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.592, "sample_count": 1675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 120108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:39:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:39:11+09:00", "last_frame_read_age_sec": 0.157, "last_frame_read_at": "2026-07-06T16:39:11+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-06T16:39:11.681198+09:00", "mtime_epoch": 1783323551.6811976, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203670}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:39:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1492.535, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.017, "pushed": 6.017, "read": 5.983}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.121, "count": 180, "max_ms": 11.032, "p95_ms": 8.833}, "gstreamer_push": {"avg_ms": 1.091, "count": 180, "max_ms": 3.234, "p95_ms": 1.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.165, "count": 180, "max_ms": 88.05, "p95_ms": 71.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.717, "count": 180, "max_ms": 118.325, "p95_ms": 105.869}, "yolo11_640_fire_11class": {"avg_ms": 62.033, "count": 180, "max_ms": 88.714, "p95_ms": 81.265}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.619, "count": 180, "max_ms": 105.301, "p95_ms": 95.666}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.719, "count": 180, "max_ms": 211.313, "p95_ms": 189.687}, "read_interval": {"avg_ms": 167.175, "count": 179, "max_ms": 220.378, "p95_ms": 198.252}, "triton_wait": {"avg_ms": 79.314, "count": 180, "max_ms": 118.338, "p95_ms": 105.881}, "visualize": {"avg_ms": 6.227, "count": 180, "max_ms": 8.099, "p95_ms": 7.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 1000}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5595.117, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 186, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:39:11+09:00", "last_error_summary": "[h264 @ 0x55b098d1a980] error while decoding MB 49 21, bytestream -13", "last_exit_at": "2026-07-06T16:39:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.512, "last_started_at": "2026-07-06T16:37:00+09:00", "pid": 17479, "process": {"exists": false, "pid": 17479}, "restart_count": 37, "restart_count_per_hour": 15, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.592, "sample_count": 1675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 120108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:39:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T16:39:11+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-06T16:39:11+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-06T16:39:11.681198+09:00", "mtime_epoch": 1783323551.6811976, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203670}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:39:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1492.535, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.016, "pushed": 6.016, "read": 5.983}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.121, "count": 180, "max_ms": 11.032, "p95_ms": 8.833}, "gstreamer_push": {"avg_ms": 1.091, "count": 180, "max_ms": 3.234, "p95_ms": 1.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.165, "count": 180, "max_ms": 88.05, "p95_ms": 71.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.717, "count": 180, "max_ms": 118.325, "p95_ms": 105.869}, "yolo11_640_fire_11class": {"avg_ms": 62.033, "count": 180, "max_ms": 88.714, "p95_ms": 81.265}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.619, "count": 180, "max_ms": 105.301, "p95_ms": 95.666}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.719, "count": 180, "max_ms": 211.313, "p95_ms": 189.687}, "read_interval": {"avg_ms": 167.175, "count": 179, "max_ms": 220.378, "p95_ms": 198.252}, "triton_wait": {"avg_ms": 79.314, "count": 180, "max_ms": 118.338, "p95_ms": 105.881}, "visualize": {"avg_ms": 6.227, "count": 180, "max_ms": 8.099, "p95_ms": 7.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 1000}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5703.387, "limit_mb": 8796093022207.996}, "detail": {"pid": 17586, "restart_count": 38, "start_count": 39}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 186, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:39:11+09:00", "last_error_summary": "[h264 @ 0x55b098d1a980] error while decoding MB 49 21, bytestream -13", "last_exit_at": "2026-07-06T16:39:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:39:14+09:00", "pid": 17586, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.32, "name": "ffmpeg", "pid": 17586, "rss_mb": 136.32, "state": "S (sleeping)", "threads": 106}, "restart_count": 38, "restart_count_per_hour": 16, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.653, "sample_count": 1644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 120108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:39:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.852, "last_frame_pushed_at": "2026-07-06T16:39:11+09:00", "last_frame_read_age_sec": 5.005, "last_frame_read_at": "2026-07-06T16:39:11+09:00", "mjpeg": {"age_sec": 4.834, "exists": true, "mtime": "2026-07-06T16:39:11.845201+09:00", "mtime_epoch": 1783323551.845201, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198984}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:39:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1753.828, "name": "celery", "pid": 237, "rss_mb": 1492.535, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 151, "pushed_frames": 151, "read_frames": 150, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.055, "pushed": 5.055, "read": 5.021}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.122, "count": 151, "max_ms": 11.032, "p95_ms": 8.833}, "gstreamer_push": {"avg_ms": 1.102, "count": 151, "max_ms": 3.234, "p95_ms": 1.52}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.905, "count": 151, "max_ms": 88.05, "p95_ms": 71.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.827, "count": 151, "max_ms": 118.325, "p95_ms": 107.369}, "yolo11_640_fire_11class": {"avg_ms": 61.962, "count": 151, "max_ms": 88.714, "p95_ms": 82.079}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.941, "count": 151, "max_ms": 105.301, "p95_ms": 96.824}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.501, "count": 151, "max_ms": 211.313, "p95_ms": 186.91}, "read_interval": {"avg_ms": 166.971, "count": 150, "max_ms": 220.378, "p95_ms": 197.286}, "triton_wait": {"avg_ms": 79.431, "count": 151, "max_ms": 118.338, "p95_ms": 107.382}, "visualize": {"avg_ms": 6.236, "count": 151, "max_ms": 8.099, "p95_ms": 7.248}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 880}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5735.949, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5568cb168480] error while decoding MB 109 21, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:40:47+09:00", "last_error_summary": "[h264 @ 0x5568cb168480] error while decoding MB 109 21, bytestream -15", "last_exit_at": "2026-07-06T16:24:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 992.116, "last_started_at": "2026-07-06T16:24:15+09:00", "pid": 17006, "process": {"exists": false, "pid": 17006}, "restart_count": 43, "restart_count_per_hour": 11, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.204, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105605, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:40:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.001, "last_frame_pushed_at": "2026-07-06T16:40:47+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-06T16:40:47+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T16:40:47.263052+09:00", "mtime_epoch": 1783323647.263052, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 237893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:40:47+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1356.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 307, "pushed_frames": 307, "read_frames": 306, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.269, "pushed": 10.269, "read": 10.235}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.008, "count": 307, "max_ms": 6.196, "p95_ms": 5.062}, "gstreamer_push": {"avg_ms": 1.438, "count": 307, "max_ms": 6.479, "p95_ms": 2.676}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.336, "count": 307, "max_ms": 85.055, "p95_ms": 69.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.352, "count": 307, "max_ms": 125.042, "p95_ms": 103.294}, "yolo11_640_fire_11class": {"avg_ms": 62.386, "count": 307, "max_ms": 96.656, "p95_ms": 82.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.344, "count": 307, "max_ms": 112.882, "p95_ms": 94.611}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.201, "count": 307, "max_ms": 137.616, "p95_ms": 116.132}, "read_interval": {"avg_ms": 97.84, "count": 306, "max_ms": 155.224, "p95_ms": 130.981}, "triton_wait": {"avg_ms": 71.562, "count": 307, "max_ms": 125.053, "p95_ms": 103.307}, "visualize": {"avg_ms": 3.635, "count": 307, "max_ms": 5.723, "p95_ms": 4.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 880}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5735.617, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:40:47+09:00", "last_error_summary": "[h264 @ 0x5568cb168480] error while decoding MB 109 21, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:40:47+09:00", "last_error_summary": "[h264 @ 0x5568cb168480] error while decoding MB 109 21, bytestream -15", "last_exit_at": "2026-07-06T16:40:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 992.134, "last_started_at": "2026-07-06T16:24:15+09:00", "pid": 17006, "process": {"exists": false, "pid": 17006}, "restart_count": 43, "restart_count_per_hour": 11, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.204, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105605, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:40:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-06T16:40:47+09:00", "last_frame_read_age_sec": 0.087, "last_frame_read_at": "2026-07-06T16:40:47+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-06T16:40:47.263052+09:00", "mtime_epoch": 1783323647.263052, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 237893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:40:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1356.16, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 307, "pushed_frames": 307, "read_frames": 306, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.264, "pushed": 10.264, "read": 10.231}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.008, "count": 307, "max_ms": 6.196, "p95_ms": 5.062}, "gstreamer_push": {"avg_ms": 1.438, "count": 307, "max_ms": 6.479, "p95_ms": 2.676}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.336, "count": 307, "max_ms": 85.055, "p95_ms": 69.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.352, "count": 307, "max_ms": 125.042, "p95_ms": 103.294}, "yolo11_640_fire_11class": {"avg_ms": 62.386, "count": 307, "max_ms": 96.656, "p95_ms": 82.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.344, "count": 307, "max_ms": 112.882, "p95_ms": 94.611}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.201, "count": 307, "max_ms": 137.616, "p95_ms": 116.132}, "read_interval": {"avg_ms": 97.84, "count": 306, "max_ms": 155.224, "p95_ms": 130.981}, "triton_wait": {"avg_ms": 71.562, "count": 307, "max_ms": 125.053, "p95_ms": 103.307}, "visualize": {"avg_ms": 3.635, "count": 307, "max_ms": 5.723, "p95_ms": 4.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 880}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5735.734, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:40:47+09:00", "last_error_summary": "[h264 @ 0x5568cb168480] error while decoding MB 109 21, bytestream -15", "last_exit_at": "2026-07-06T16:40:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 992.144, "last_started_at": "2026-07-06T16:24:15+09:00", "pid": 17006, "process": {"exists": false, "pid": 17006}, "restart_count": 43, "restart_count_per_hour": 11, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.204, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105605, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:40:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.029, "last_frame_pushed_at": "2026-07-06T16:40:47+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-06T16:40:47+09:00", "mjpeg": {"age_sec": 0.01, "exists": true, "mtime": "2026-07-06T16:40:47.335053+09:00", "mtime_epoch": 1783323647.3350532, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236031}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:40:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1356.16, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 307, "pushed_frames": 307, "read_frames": 306, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.261, "pushed": 10.261, "read": 10.227}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.008, "count": 307, "max_ms": 6.196, "p95_ms": 5.062}, "gstreamer_push": {"avg_ms": 1.438, "count": 307, "max_ms": 6.479, "p95_ms": 2.676}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.336, "count": 307, "max_ms": 85.055, "p95_ms": 69.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.352, "count": 307, "max_ms": 125.042, "p95_ms": 103.294}, "yolo11_640_fire_11class": {"avg_ms": 62.386, "count": 307, "max_ms": 96.656, "p95_ms": 82.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.344, "count": 307, "max_ms": 112.882, "p95_ms": 94.611}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.201, "count": 307, "max_ms": 137.616, "p95_ms": 116.132}, "read_interval": {"avg_ms": 97.84, "count": 306, "max_ms": 155.224, "p95_ms": 130.981}, "triton_wait": {"avg_ms": 71.562, "count": 307, "max_ms": 125.053, "p95_ms": 103.307}, "visualize": {"avg_ms": 3.635, "count": 307, "max_ms": 5.723, "p95_ms": 4.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 880}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5745.672, "limit_mb": 8796093022207.996}, "detail": {"pid": 17753, "restart_count": 44, "start_count": 45}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:40:47+09:00", "last_error_summary": "[h264 @ 0x5568cb168480] error while decoding MB 109 21, bytestream -15", "last_exit_at": "2026-07-06T16:40:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:40:49+09:00", "pid": 17753, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.172, "name": "ffmpeg", "pid": 17753, "rss_mb": 58.172, "state": "S (sleeping)", "threads": 1}, "restart_count": 44, "restart_count_per_hour": 12, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.204, "sample_count": 2467, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105605, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:40:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.175, "last_frame_pushed_at": "2026-07-06T16:40:47+09:00", "last_frame_read_age_sec": 4.242, "last_frame_read_at": "2026-07-06T16:40:47+09:00", "mjpeg": {"age_sec": 4.156, "exists": true, "mtime": "2026-07-06T16:40:47.335053+09:00", "mtime_epoch": 1783323647.3350532, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236031}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:40:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1356.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 264, "pushed_frames": 264, "read_frames": 263, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.807, "pushed": 8.807, "read": 8.774}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.036, "count": 264, "max_ms": 6.196, "p95_ms": 5.154}, "gstreamer_push": {"avg_ms": 1.466, "count": 264, "max_ms": 6.479, "p95_ms": 3.404}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.222, "count": 264, "max_ms": 85.055, "p95_ms": 69.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.524, "count": 264, "max_ms": 125.042, "p95_ms": 105.014}, "yolo11_640_fire_11class": {"avg_ms": 62.372, "count": 264, "max_ms": 96.656, "p95_ms": 81.512}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.104, "count": 264, "max_ms": 112.882, "p95_ms": 93.984}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.389, "count": 264, "max_ms": 137.616, "p95_ms": 117.843}, "read_interval": {"avg_ms": 98.113, "count": 263, "max_ms": 155.224, "p95_ms": 130.981}, "triton_wait": {"avg_ms": 71.68, "count": 264, "max_ms": 125.053, "p95_ms": 105.026}, "visualize": {"avg_ms": 3.653, "count": 264, "max_ms": 5.723, "p95_ms": 4.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5770.957, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55648c393780] error while decoding MB 3 23, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 187, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:41:23+09:00", "last_error_summary": "[h264 @ 0x55648c393780] error while decoding MB 3 23, bytestream -11", "last_exit_at": "2026-07-06T16:39:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.292, "last_started_at": "2026-07-06T16:39:14+09:00", "pid": 17586, "process": {"exists": false, "pid": 17586}, "restart_count": 38, "restart_count_per_hour": 16, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.707, "sample_count": 1592, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 120745, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:41:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T16:41:23+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T16:41:23+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T16:41:23.931709+09:00", "mtime_epoch": 1783323683.9317088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196002}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:41:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1671.52, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.467, "pushed": 4.467, "read": 4.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.255, "count": 133, "max_ms": 18.649, "p95_ms": 11.555}, "gstreamer_push": {"avg_ms": 0.88, "count": 133, "max_ms": 1.322, "p95_ms": 0.968}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.688, "count": 133, "max_ms": 80.385, "p95_ms": 75.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.667, "count": 133, "max_ms": 121.93, "p95_ms": 104.826}, "yolo11_640_fire_11class": {"avg_ms": 62.271, "count": 133, "max_ms": 86.639, "p95_ms": 80.753}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.574, "count": 133, "max_ms": 105.101, "p95_ms": 99.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.992, "count": 133, "max_ms": 267.543, "p95_ms": 256.349}, "read_interval": {"avg_ms": 225.308, "count": 133, "max_ms": 276.559, "p95_ms": 262.953}, "triton_wait": {"avg_ms": 80.797, "count": 133, "max_ms": 121.948, "p95_ms": 104.838}, "visualize": {"avg_ms": 10.625, "count": 133, "max_ms": 17.622, "p95_ms": 14.648}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.777, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5761.008, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:41:23+09:00", "last_error_summary": "[h264 @ 0x55648c393780] error while decoding MB 3 23, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 187, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:41:23+09:00", "last_error_summary": "[h264 @ 0x55648c393780] error while decoding MB 3 23, bytestream -11", "last_exit_at": "2026-07-06T16:41:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.461, "last_started_at": "2026-07-06T16:39:14+09:00", "pid": 17586, "process": {"exists": false, "pid": 17586}, "restart_count": 38, "restart_count_per_hour": 16, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.707, "sample_count": 1592, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 120746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:41:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T16:41:24+09:00", "last_frame_read_age_sec": 0.219, "last_frame_read_at": "2026-07-06T16:41:23+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-06T16:41:23.931709+09:00", "mtime_epoch": 1783323683.9317088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196002}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:41:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1671.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.475, "pushed": 4.475, "read": 4.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.248, "count": 133, "max_ms": 18.649, "p95_ms": 11.555}, "gstreamer_push": {"avg_ms": 0.88, "count": 134, "max_ms": 1.322, "p95_ms": 0.968}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.597, "count": 133, "max_ms": 80.385, "p95_ms": 75.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.402, "count": 133, "max_ms": 121.93, "p95_ms": 104.826}, "yolo11_640_fire_11class": {"avg_ms": 62.172, "count": 133, "max_ms": 86.639, "p95_ms": 80.753}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.498, "count": 133, "max_ms": 105.101, "p95_ms": 99.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.968, "count": 134, "max_ms": 267.543, "p95_ms": 256.349}, "read_interval": {"avg_ms": 225.308, "count": 133, "max_ms": 276.559, "p95_ms": 262.953}, "triton_wait": {"avg_ms": 80.532, "count": 133, "max_ms": 121.948, "p95_ms": 104.838}, "visualize": {"avg_ms": 10.598, "count": 133, "max_ms": 17.622, "p95_ms": 14.648}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5761.125, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 187, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:41:23+09:00", "last_error_summary": "[h264 @ 0x55648c393780] error while decoding MB 3 23, bytestream -11", "last_exit_at": "2026-07-06T16:41:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.465, "last_started_at": "2026-07-06T16:39:14+09:00", "pid": 17586, "process": {"exists": false, "pid": 17586}, "restart_count": 38, "restart_count_per_hour": 16, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.707, "sample_count": 1592, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 120746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:41:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T16:41:24+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-06T16:41:23+09:00", "mjpeg": {"age_sec": 0.212, "exists": true, "mtime": "2026-07-06T16:41:23.931709+09:00", "mtime_epoch": 1783323683.9317088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196002}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:41:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1671.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.474, "pushed": 4.474, "read": 4.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.248, "count": 133, "max_ms": 18.649, "p95_ms": 11.555}, "gstreamer_push": {"avg_ms": 0.88, "count": 134, "max_ms": 1.322, "p95_ms": 0.968}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.597, "count": 133, "max_ms": 80.385, "p95_ms": 75.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.402, "count": 133, "max_ms": 121.93, "p95_ms": 104.826}, "yolo11_640_fire_11class": {"avg_ms": 62.172, "count": 133, "max_ms": 86.639, "p95_ms": 80.753}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.498, "count": 133, "max_ms": 105.101, "p95_ms": 99.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.968, "count": 134, "max_ms": 267.543, "p95_ms": 256.349}, "read_interval": {"avg_ms": 225.308, "count": 133, "max_ms": 276.559, "p95_ms": 262.953}, "triton_wait": {"avg_ms": 80.532, "count": 133, "max_ms": 121.948, "p95_ms": 104.838}, "visualize": {"avg_ms": 10.598, "count": 133, "max_ms": 17.622, "p95_ms": 14.648}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5875.031, "limit_mb": 8796093022207.996}, "detail": {"pid": 17905, "restart_count": 39, "start_count": 40}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 187, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:41:23+09:00", "last_error_summary": "[h264 @ 0x55648c393780] error while decoding MB 3 23, bytestream -11", "last_exit_at": "2026-07-06T16:41:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:41:26+09:00", "pid": 17905, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.352, "name": "ffmpeg", "pid": 17905, "rss_mb": 139.352, "state": "S (sleeping)", "threads": 106}, "restart_count": 39, "restart_count_per_hour": 17, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.7, "sample_count": 1572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 120746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:41:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.447, "last_frame_pushed_at": "2026-07-06T16:41:24+09:00", "last_frame_read_age_sec": 4.663, "last_frame_read_at": "2026-07-06T16:41:23+09:00", "mjpeg": {"age_sec": 4.429, "exists": true, "mtime": "2026-07-06T16:41:24.155709+09:00", "mtime_epoch": 1783323684.1557093, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195063}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:41:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1671.52, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.805, "pushed": 3.805, "read": 3.771}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.269, "count": 113, "max_ms": 18.649, "p95_ms": 11.555}, "gstreamer_push": {"avg_ms": 0.882, "count": 114, "max_ms": 1.322, "p95_ms": 0.968}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.974, "count": 113, "max_ms": 80.385, "p95_ms": 75.323}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.037, "count": 113, "max_ms": 121.93, "p95_ms": 104.965}, "yolo11_640_fire_11class": {"avg_ms": 62.483, "count": 113, "max_ms": 86.639, "p95_ms": 82.565}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.027, "count": 113, "max_ms": 105.101, "p95_ms": 99.704}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.385, "count": 114, "max_ms": 267.543, "p95_ms": 257.924}, "read_interval": {"avg_ms": 225.74, "count": 113, "max_ms": 276.559, "p95_ms": 264.081}, "triton_wait": {"avg_ms": 81.187, "count": 113, "max_ms": 121.948, "p95_ms": 104.978}, "visualize": {"avg_ms": 10.064, "count": 113, "max_ms": 17.622, "p95_ms": 14.46}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1017}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5720.074, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a053a11080] error while decoding MB 51 30, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:44:40+09:00", "last_error_summary": "[h264 @ 0x55a053a11080] error while decoding MB 51 30, bytestream -17", "last_exit_at": "2026-07-06T16:41:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.857, "last_started_at": "2026-07-06T16:41:26+09:00", "pid": 17905, "process": {"exists": false, "pid": 17905}, "restart_count": 39, "restart_count_per_hour": 16, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.458, "sample_count": 1930, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 122144, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:44:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T16:44:40+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T16:44:40+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T16:44:40.396532+09:00", "mtime_epoch": 1783323880.3965318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196383}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:44:40+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.824, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.458, "pushed": 7.458, "read": 7.458}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.744, "count": 223, "max_ms": 8.987, "p95_ms": 6.76}, "gstreamer_push": {"avg_ms": 1.162, "count": 223, "max_ms": 1.529, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.505, "count": 223, "max_ms": 88.51, "p95_ms": 77.97}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.745, "count": 223, "max_ms": 125.345, "p95_ms": 105.437}, "yolo11_640_fire_11class": {"avg_ms": 63.384, "count": 223, "max_ms": 93.805, "p95_ms": 86.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.636, "count": 223, "max_ms": 107.627, "p95_ms": 95.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.648, "count": 223, "max_ms": 178.513, "p95_ms": 156.565}, "read_interval": {"avg_ms": 134.511, "count": 223, "max_ms": 187.268, "p95_ms": 165.378}, "triton_wait": {"avg_ms": 77.853, "count": 223, "max_ms": 125.382, "p95_ms": 105.451}, "visualize": {"avg_ms": 4.97, "count": 223, "max_ms": 6.597, "p95_ms": 5.712}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1017}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5719.113, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:44:40+09:00", "last_error_summary": "[h264 @ 0x55a053a11080] error while decoding MB 51 30, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:44:40+09:00", "last_error_summary": "[h264 @ 0x55a053a11080] error while decoding MB 51 30, bytestream -17", "last_exit_at": "2026-07-06T16:44:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.94, "last_started_at": "2026-07-06T16:41:26+09:00", "pid": 17905, "process": {"exists": false, "pid": 17905}, "restart_count": 39, "restart_count_per_hour": 16, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.176, "sample_count": 1930, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 122145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:44:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T16:44:40+09:00", "last_frame_read_age_sec": 0.135, "last_frame_read_at": "2026-07-06T16:44:40+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-06T16:44:40.396532+09:00", "mtime_epoch": 1783323880.3965318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196383}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:44:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.824, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 224, "pushed_frames": 224, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.47, "pushed": 7.47, "read": 7.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.746, "count": 223, "max_ms": 8.987, "p95_ms": 6.76}, "gstreamer_push": {"avg_ms": 1.162, "count": 224, "max_ms": 1.529, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.471, "count": 223, "max_ms": 88.51, "p95_ms": 77.97}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.661, "count": 223, "max_ms": 125.345, "p95_ms": 105.437}, "yolo11_640_fire_11class": {"avg_ms": 63.314, "count": 223, "max_ms": 93.805, "p95_ms": 86.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.519, "count": 223, "max_ms": 107.627, "p95_ms": 95.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.665, "count": 224, "max_ms": 178.513, "p95_ms": 156.565}, "read_interval": {"avg_ms": 134.511, "count": 223, "max_ms": 187.268, "p95_ms": 165.378}, "triton_wait": {"avg_ms": 77.769, "count": 223, "max_ms": 125.382, "p95_ms": 105.451}, "visualize": {"avg_ms": 4.97, "count": 223, "max_ms": 6.597, "p95_ms": 5.712}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1017}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5719.25, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:44:40+09:00", "last_error_summary": "[h264 @ 0x55a053a11080] error while decoding MB 51 30, bytestream -17", "last_exit_at": "2026-07-06T16:44:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.946, "last_started_at": "2026-07-06T16:41:26+09:00", "pid": 17905, "process": {"exists": false, "pid": 17905}, "restart_count": 39, "restart_count_per_hour": 16, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.176, "sample_count": 1930, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 122145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:44:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T16:44:40+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-06T16:44:40+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-06T16:44:40.396532+09:00", "mtime_epoch": 1783323880.3965318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196383}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:44:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.824, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 224, "pushed_frames": 224, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.469, "pushed": 7.469, "read": 7.436}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.746, "count": 223, "max_ms": 8.987, "p95_ms": 6.76}, "gstreamer_push": {"avg_ms": 1.162, "count": 224, "max_ms": 1.529, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.471, "count": 223, "max_ms": 88.51, "p95_ms": 77.97}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.661, "count": 223, "max_ms": 125.345, "p95_ms": 105.437}, "yolo11_640_fire_11class": {"avg_ms": 63.314, "count": 223, "max_ms": 93.805, "p95_ms": 86.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.519, "count": 223, "max_ms": 107.627, "p95_ms": 95.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.665, "count": 224, "max_ms": 178.513, "p95_ms": 156.565}, "read_interval": {"avg_ms": 134.511, "count": 223, "max_ms": 187.268, "p95_ms": 165.378}, "triton_wait": {"avg_ms": 77.769, "count": 223, "max_ms": 125.382, "p95_ms": 105.451}, "visualize": {"avg_ms": 4.97, "count": 223, "max_ms": 6.597, "p95_ms": 5.712}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1017}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5817.27, "limit_mb": 8796093022207.996}, "detail": {"pid": 18012, "restart_count": 40, "start_count": 41}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:44:40+09:00", "last_error_summary": "[h264 @ 0x55a053a11080] error while decoding MB 51 30, bytestream -17", "last_exit_at": "2026-07-06T16:44:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:44:42+09:00", "pid": 18012, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.496, "name": "ffmpeg", "pid": 18012, "rss_mb": 136.496, "state": "S (sleeping)", "threads": 106}, "restart_count": 40, "restart_count_per_hour": 17, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.92, "sample_count": 1915, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 122145, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:44:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.028, "last_frame_pushed_at": "2026-07-06T16:44:40+09:00", "last_frame_read_age_sec": 4.159, "last_frame_read_at": "2026-07-06T16:44:40+09:00", "mjpeg": {"age_sec": 4.014, "exists": true, "mtime": "2026-07-06T16:44:40.532533+09:00", "mtime_epoch": 1783323880.5325327, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196558}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:44:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.824, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 193, "pushed_frames": 193, "read_frames": 192, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.443, "pushed": 6.443, "read": 6.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.736, "count": 193, "max_ms": 8.987, "p95_ms": 6.652}, "gstreamer_push": {"avg_ms": 1.161, "count": 193, "max_ms": 1.529, "p95_ms": 1.362}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.662, "count": 193, "max_ms": 88.51, "p95_ms": 79.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.174, "count": 193, "max_ms": 125.345, "p95_ms": 106.516}, "yolo11_640_fire_11class": {"avg_ms": 63.561, "count": 193, "max_ms": 93.805, "p95_ms": 87.041}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.85, "count": 193, "max_ms": 107.627, "p95_ms": 96.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.119, "count": 193, "max_ms": 178.513, "p95_ms": 157.571}, "read_interval": {"avg_ms": 134.96, "count": 192, "max_ms": 187.268, "p95_ms": 166.048}, "triton_wait": {"avg_ms": 78.254, "count": 193, "max_ms": 125.382, "p95_ms": 106.528}, "visualize": {"avg_ms": 4.982, "count": 193, "max_ms": 6.597, "p95_ms": 5.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1030}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5726.359, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560242335880] error while decoding MB 71 19, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 189, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:48:12+09:00", "last_error_summary": "[h264 @ 0x560242335880] error while decoding MB 71 19, bytestream -26", "last_exit_at": "2026-07-06T16:44:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.791, "last_started_at": "2026-07-06T16:44:42+09:00", "pid": 18012, "process": {"exists": false, "pid": 18012}, "restart_count": 40, "restart_count_per_hour": 17, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.788, "sample_count": 2198, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 123695, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:48:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.053, "last_frame_pushed_at": "2026-07-06T16:48:12+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-06T16:48:12+09:00", "mjpeg": {"age_sec": 0.034, "exists": true, "mtime": "2026-07-06T16:48:12.302325+09:00", "mtime_epoch": 1783324092.3023248, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197194}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:48:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.598, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 228, "pushed_frames": 229, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.6, "pushed": 7.634, "read": 7.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.367, "count": 228, "max_ms": 9.523, "p95_ms": 6.992}, "gstreamer_push": {"avg_ms": 1.167, "count": 229, "max_ms": 1.654, "p95_ms": 1.366}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.255, "count": 228, "max_ms": 87.073, "p95_ms": 77.701}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.637, "count": 228, "max_ms": 120.029, "p95_ms": 106.588}, "yolo11_640_fire_11class": {"avg_ms": 64.086, "count": 228, "max_ms": 93.961, "p95_ms": 84.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.003, "count": 228, "max_ms": 103.494, "p95_ms": 94.832}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.819, "count": 228, "max_ms": 169.266, "p95_ms": 156.347}, "read_interval": {"avg_ms": 131.333, "count": 229, "max_ms": 178.706, "p95_ms": 165.705}, "triton_wait": {"avg_ms": 79.812, "count": 228, "max_ms": 120.042, "p95_ms": 106.605}, "visualize": {"avg_ms": 4.718, "count": 228, "max_ms": 8.984, "p95_ms": 5.696}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1030}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5728.203, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:48:12+09:00", "last_error_summary": "[h264 @ 0x560242335880] error while decoding MB 71 19, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 189, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:48:12+09:00", "last_error_summary": "[h264 @ 0x560242335880] error while decoding MB 71 19, bytestream -26", "last_exit_at": "2026-07-06T16:48:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.889, "last_started_at": "2026-07-06T16:44:42+09:00", "pid": 18012, "process": {"exists": false, "pid": 18012}, "restart_count": 40, "restart_count_per_hour": 17, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.788, "sample_count": 2199, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 123696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:48:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T16:48:12+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T16:48:12+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-06T16:48:12.302325+09:00", "mtime_epoch": 1783324092.3023248, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197194}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:48:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.598, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.639, "pushed": 7.639, "read": 7.605}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.366, "count": 228, "max_ms": 9.523, "p95_ms": 6.992}, "gstreamer_push": {"avg_ms": 1.167, "count": 229, "max_ms": 1.654, "p95_ms": 1.366}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.405, "count": 228, "max_ms": 87.073, "p95_ms": 77.701}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.794, "count": 228, "max_ms": 120.029, "p95_ms": 106.588}, "yolo11_640_fire_11class": {"avg_ms": 64.2, "count": 228, "max_ms": 93.961, "p95_ms": 84.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.132, "count": 228, "max_ms": 103.494, "p95_ms": 94.832}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.884, "count": 229, "max_ms": 169.266, "p95_ms": 156.347}, "read_interval": {"avg_ms": 131.338, "count": 228, "max_ms": 178.706, "p95_ms": 165.705}, "triton_wait": {"avg_ms": 79.969, "count": 228, "max_ms": 120.042, "p95_ms": 106.605}, "visualize": {"avg_ms": 4.711, "count": 228, "max_ms": 8.984, "p95_ms": 5.696}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1030}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5728.32, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 189, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:48:12+09:00", "last_error_summary": "[h264 @ 0x560242335880] error while decoding MB 71 19, bytestream -26", "last_exit_at": "2026-07-06T16:48:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.896, "last_started_at": "2026-07-06T16:44:42+09:00", "pid": 18012, "process": {"exists": false, "pid": 18012}, "restart_count": 40, "restart_count_per_hour": 17, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.788, "sample_count": 2198, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 123696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:48:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T16:48:12+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T16:48:12+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-06T16:48:12.302325+09:00", "mtime_epoch": 1783324092.3023248, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197194}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:48:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.598, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.637, "pushed": 7.637, "read": 7.604}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.366, "count": 228, "max_ms": 9.523, "p95_ms": 6.992}, "gstreamer_push": {"avg_ms": 1.167, "count": 229, "max_ms": 1.654, "p95_ms": 1.366}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.405, "count": 228, "max_ms": 87.073, "p95_ms": 77.701}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.794, "count": 228, "max_ms": 120.029, "p95_ms": 106.588}, "yolo11_640_fire_11class": {"avg_ms": 64.2, "count": 228, "max_ms": 93.961, "p95_ms": 84.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.132, "count": 228, "max_ms": 103.494, "p95_ms": 94.832}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.884, "count": 229, "max_ms": 169.266, "p95_ms": 156.347}, "read_interval": {"avg_ms": 131.338, "count": 228, "max_ms": 178.706, "p95_ms": 165.705}, "triton_wait": {"avg_ms": 79.969, "count": 228, "max_ms": 120.042, "p95_ms": 106.605}, "visualize": {"avg_ms": 4.711, "count": 228, "max_ms": 8.984, "p95_ms": 5.696}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1030}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5836.289, "limit_mb": 8796093022207.996}, "detail": {"pid": 18119, "restart_count": 41, "start_count": 42}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 189, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:48:12+09:00", "last_error_summary": "[h264 @ 0x560242335880] error while decoding MB 71 19, bytestream -26", "last_exit_at": "2026-07-06T16:48:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:48:14+09:00", "pid": 18119, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.336, "name": "ffmpeg", "pid": 18119, "rss_mb": 139.336, "state": "S (sleeping)", "threads": 106}, "restart_count": 41, "restart_count_per_hour": 18, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.565, "sample_count": 2166, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 123696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:48:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.332, "last_frame_pushed_at": "2026-07-06T16:48:12+09:00", "last_frame_read_age_sec": 4.47, "last_frame_read_at": "2026-07-06T16:48:12+09:00", "mjpeg": {"age_sec": 4.314, "exists": true, "mtime": "2026-07-06T16:48:12.446326+09:00", "mtime_epoch": 1783324092.446326, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197361}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:48:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.598, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.567, "pushed": 6.6, "read": 6.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.26, "count": 197, "max_ms": 9.523, "p95_ms": 6.992}, "gstreamer_push": {"avg_ms": 1.174, "count": 198, "max_ms": 1.654, "p95_ms": 1.379}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.935, "count": 197, "max_ms": 87.073, "p95_ms": 78.255}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.407, "count": 197, "max_ms": 120.029, "p95_ms": 107.048}, "yolo11_640_fire_11class": {"avg_ms": 64.712, "count": 197, "max_ms": 93.961, "p95_ms": 84.965}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.714, "count": 197, "max_ms": 103.494, "p95_ms": 96.258}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.568, "count": 197, "max_ms": 169.266, "p95_ms": 156.411}, "read_interval": {"avg_ms": 130.33, "count": 197, "max_ms": 178.706, "p95_ms": 166.477}, "triton_wait": {"avg_ms": 80.572, "count": 197, "max_ms": 120.042, "p95_ms": 107.06}, "visualize": {"avg_ms": 4.64, "count": 197, "max_ms": 8.984, "p95_ms": 5.806}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5739.305, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b31f055c80] error while decoding MB 79 12, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 190, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:51:30+09:00", "last_error_summary": "[h264 @ 0x55b31f055c80] error while decoding MB 79 12, bytestream -9", "last_exit_at": "2026-07-06T16:48:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.389, "last_started_at": "2026-07-06T16:48:14+09:00", "pid": 18119, "process": {"exists": false, "pid": 18119}, "restart_count": 41, "restart_count_per_hour": 17, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.829, "sample_count": 2181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 125118, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:51:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T16:51:30+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T16:51:30+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-06T16:51:30.096569+09:00", "mtime_epoch": 1783324290.096569, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197562}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:51:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.152, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.535, "pushed": 7.535, "read": 7.535}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.873, "count": 225, "max_ms": 9.286, "p95_ms": 7.593}, "gstreamer_push": {"avg_ms": 1.086, "count": 226, "max_ms": 1.411, "p95_ms": 1.297}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.123, "count": 225, "max_ms": 93.263, "p95_ms": 79.937}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.092, "count": 225, "max_ms": 134.341, "p95_ms": 103.61}, "yolo11_640_fire_11class": {"avg_ms": 57.912, "count": 225, "max_ms": 90.328, "p95_ms": 78.395}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.429, "count": 225, "max_ms": 114.275, "p95_ms": 98.779}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.012, "count": 226, "max_ms": 187.281, "p95_ms": 157.083}, "read_interval": {"avg_ms": 132.93, "count": 226, "max_ms": 198.395, "p95_ms": 167.593}, "triton_wait": {"avg_ms": 73.415, "count": 225, "max_ms": 134.354, "p95_ms": 105.006}, "visualize": {"avg_ms": 4.982, "count": 225, "max_ms": 6.945, "p95_ms": 6.069}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5738.887, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:51:30+09:00", "last_error_summary": "[h264 @ 0x55b31f055c80] error while decoding MB 79 12, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 190, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:51:30+09:00", "last_error_summary": "[h264 @ 0x55b31f055c80] error while decoding MB 79 12, bytestream -9", "last_exit_at": "2026-07-06T16:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.457, "last_started_at": "2026-07-06T16:48:14+09:00", "pid": 18119, "process": {"exists": false, "pid": 18119}, "restart_count": 41, "restart_count_per_hour": 17, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.829, "sample_count": 2181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 125119, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:51:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T16:51:30+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T16:51:30+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T16:51:30.096569+09:00", "mtime_epoch": 1783324290.096569, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197562}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:51:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.152, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.552, "pushed": 7.552, "read": 7.519}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.886, "count": 226, "max_ms": 9.286, "p95_ms": 7.752}, "gstreamer_push": {"avg_ms": 1.085, "count": 226, "max_ms": 1.411, "p95_ms": 1.297}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.078, "count": 226, "max_ms": 93.263, "p95_ms": 79.937}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.062, "count": 226, "max_ms": 134.341, "p95_ms": 103.61}, "yolo11_640_fire_11class": {"avg_ms": 57.9, "count": 226, "max_ms": 90.328, "p95_ms": 78.395}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.407, "count": 226, "max_ms": 114.275, "p95_ms": 98.779}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.074, "count": 226, "max_ms": 187.281, "p95_ms": 157.083}, "read_interval": {"avg_ms": 133.001, "count": 225, "max_ms": 198.395, "p95_ms": 167.593}, "triton_wait": {"avg_ms": 73.383, "count": 226, "max_ms": 134.354, "p95_ms": 105.006}, "visualize": {"avg_ms": 4.991, "count": 226, "max_ms": 7.017, "p95_ms": 6.082}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5738.926, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 190, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:51:30+09:00", "last_error_summary": "[h264 @ 0x55b31f055c80] error while decoding MB 79 12, bytestream -9", "last_exit_at": "2026-07-06T16:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.464, "last_started_at": "2026-07-06T16:48:14+09:00", "pid": 18119, "process": {"exists": false, "pid": 18119}, "restart_count": 41, "restart_count_per_hour": 17, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.829, "sample_count": 2181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 125119, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:51:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T16:51:30+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T16:51:30+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-06T16:51:30.096569+09:00", "mtime_epoch": 1783324290.096569, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197562}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:51:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.152, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 226, "read_frames": 225, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.551, "pushed": 7.551, "read": 7.517}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.886, "count": 226, "max_ms": 9.286, "p95_ms": 7.752}, "gstreamer_push": {"avg_ms": 1.085, "count": 226, "max_ms": 1.411, "p95_ms": 1.297}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.078, "count": 226, "max_ms": 93.263, "p95_ms": 79.937}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.062, "count": 226, "max_ms": 134.341, "p95_ms": 103.61}, "yolo11_640_fire_11class": {"avg_ms": 57.9, "count": 226, "max_ms": 90.328, "p95_ms": 78.395}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.407, "count": 226, "max_ms": 114.275, "p95_ms": 98.779}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.074, "count": 226, "max_ms": 187.281, "p95_ms": 157.083}, "read_interval": {"avg_ms": 133.001, "count": 225, "max_ms": 198.395, "p95_ms": 167.593}, "triton_wait": {"avg_ms": 73.383, "count": 226, "max_ms": 134.354, "p95_ms": 105.006}, "visualize": {"avg_ms": 4.991, "count": 226, "max_ms": 7.017, "p95_ms": 6.082}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5847.215, "limit_mb": 8796093022207.996}, "detail": {"pid": 18226, "restart_count": 42, "start_count": 43}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 190, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:51:30+09:00", "last_error_summary": "[h264 @ 0x55b31f055c80] error while decoding MB 79 12, bytestream -9", "last_exit_at": "2026-07-06T16:51:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:51:32+09:00", "pid": 18226, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.125, "name": "ffmpeg", "pid": 18226, "rss_mb": 139.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 42, "restart_count_per_hour": 18, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.008, "sample_count": 2147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 125119, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:51:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.328, "last_frame_pushed_at": "2026-07-06T16:51:30+09:00", "last_frame_read_age_sec": 4.453, "last_frame_read_at": "2026-07-06T16:51:30+09:00", "mjpeg": {"age_sec": 4.312, "exists": true, "mtime": "2026-07-06T16:51:30.228571+09:00", "mtime_epoch": 1783324290.2285707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197743}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:51:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.152, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 193, "pushed_frames": 193, "read_frames": 192, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.46, "pushed": 6.46, "read": 6.427}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.826, "count": 193, "max_ms": 9.286, "p95_ms": 7.5}, "gstreamer_push": {"avg_ms": 1.093, "count": 193, "max_ms": 1.411, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.195, "count": 193, "max_ms": 93.263, "p95_ms": 80.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.732, "count": 193, "max_ms": 134.341, "p95_ms": 104.382}, "yolo11_640_fire_11class": {"avg_ms": 58.375, "count": 193, "max_ms": 90.328, "p95_ms": 78.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.975, "count": 193, "max_ms": 114.275, "p95_ms": 100.536}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.055, "count": 193, "max_ms": 187.281, "p95_ms": 158.143}, "read_interval": {"avg_ms": 133.066, "count": 192, "max_ms": 198.395, "p95_ms": 167.955}, "triton_wait": {"avg_ms": 73.973, "count": 193, "max_ms": 134.354, "p95_ms": 105.149}, "visualize": {"avg_ms": 4.959, "count": 193, "max_ms": 7.017, "p95_ms": 6.069}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1050}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5706.129, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:53:59+09:00", "last_error_summary": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "last_exit_at": "2026-07-06T16:51:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.667, "last_started_at": "2026-07-06T16:51:32+09:00", "pid": 18226, "process": {"exists": false, "pid": 18226}, "restart_count": 42, "restart_count_per_hour": 18, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.759, "sample_count": 2010, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 126013, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:53:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T16:53:59+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T16:53:59+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T16:53:59.162513+09:00", "mtime_epoch": 1783324439.1625128, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197357}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:53:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1597.441, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 80, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.673, "pushed": 2.673, "read": 2.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.255, "count": 80, "max_ms": 8.838, "p95_ms": 8.092}, "gstreamer_push": {"avg_ms": 1.124, "count": 80, "max_ms": 1.271, "p95_ms": 1.233}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.625, "count": 80, "max_ms": 77.949, "p95_ms": 74.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.467, "count": 80, "max_ms": 111.575, "p95_ms": 102.334}, "yolo11_640_fire_11class": {"avg_ms": 62.094, "count": 80, "max_ms": 87.457, "p95_ms": 80.544}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.158, "count": 80, "max_ms": 95.051, "p95_ms": 89.968}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.054, "count": 80, "max_ms": 166.301, "p95_ms": 155.433}, "read_interval": {"avg_ms": 374.782, "count": 80, "max_ms": 18851.957, "p95_ms": 167.165}, "triton_wait": {"avg_ms": 77.48, "count": 80, "max_ms": 111.588, "p95_ms": 102.347}, "visualize": {"avg_ms": 5.168, "count": 80, "max_ms": 6.386, "p95_ms": 5.668}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1050}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5705.508, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:53:59+09:00", "last_error_summary": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:53:59+09:00", "last_error_summary": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "last_exit_at": "2026-07-06T16:53:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.736, "last_started_at": "2026-07-06T16:51:32+09:00", "pid": 18226, "process": {"exists": false, "pid": 18226}, "restart_count": 42, "restart_count_per_hour": 18, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.759, "sample_count": 2011, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 126014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:53:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T16:53:59+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T16:53:59+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T16:53:59.162513+09:00", "mtime_epoch": 1783324439.1625128, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197357}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:53:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1597.441, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.7, "pushed": 2.7, "read": 2.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.301, "count": 80, "max_ms": 9.509, "p95_ms": 8.135}, "gstreamer_push": {"avg_ms": 1.123, "count": 81, "max_ms": 1.271, "p95_ms": 1.233}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.649, "count": 80, "max_ms": 77.949, "p95_ms": 74.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.48, "count": 80, "max_ms": 111.575, "p95_ms": 102.334}, "yolo11_640_fire_11class": {"avg_ms": 62.081, "count": 80, "max_ms": 87.457, "p95_ms": 80.544}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.206, "count": 80, "max_ms": 95.051, "p95_ms": 89.968}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.912, "count": 81, "max_ms": 166.301, "p95_ms": 155.433}, "read_interval": {"avg_ms": 374.782, "count": 80, "max_ms": 18851.957, "p95_ms": 167.165}, "triton_wait": {"avg_ms": 77.493, "count": 80, "max_ms": 111.588, "p95_ms": 102.347}, "visualize": {"avg_ms": 5.175, "count": 80, "max_ms": 6.386, "p95_ms": 5.668}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1050}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5705.5, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:53:59+09:00", "last_error_summary": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "last_exit_at": "2026-07-06T16:53:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.741, "last_started_at": "2026-07-06T16:51:32+09:00", "pid": 18226, "process": {"exists": false, "pid": 18226}, "restart_count": 42, "restart_count_per_hour": 18, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.759, "sample_count": 2011, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 126014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:53:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T16:53:59+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-06T16:53:59+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-06T16:53:59.162513+09:00", "mtime_epoch": 1783324439.1625128, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197357}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:53:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1597.441, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 80, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.667, "pushed": 2.667, "read": 2.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.301, "count": 80, "max_ms": 9.509, "p95_ms": 8.135}, "gstreamer_push": {"avg_ms": 1.124, "count": 80, "max_ms": 1.271, "p95_ms": 1.233}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.649, "count": 80, "max_ms": 77.949, "p95_ms": 74.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.48, "count": 80, "max_ms": 111.575, "p95_ms": 102.334}, "yolo11_640_fire_11class": {"avg_ms": 62.081, "count": 80, "max_ms": 87.457, "p95_ms": 80.544}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.206, "count": 80, "max_ms": 95.051, "p95_ms": 89.968}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.166, "count": 80, "max_ms": 166.301, "p95_ms": 155.433}, "read_interval": {"avg_ms": 374.782, "count": 80, "max_ms": 18851.957, "p95_ms": 167.165}, "triton_wait": {"avg_ms": 77.493, "count": 80, "max_ms": 111.588, "p95_ms": 102.347}, "visualize": {"avg_ms": 5.175, "count": 80, "max_ms": 6.386, "p95_ms": 5.668}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1050}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5805.07, "limit_mb": 8796093022207.996}, "detail": {"pid": 18333, "restart_count": 43, "start_count": 44}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:53:59+09:00", "last_error_summary": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "last_exit_at": "2026-07-06T16:54:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:54:01+09:00", "pid": 18333, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.082, "name": "ffmpeg", "pid": 18333, "rss_mb": 139.082, "state": "S (sleeping)", "threads": 106}, "restart_count": 43, "restart_count_per_hour": 19, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.65, "sample_count": 1976, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 126014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:53:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.686, "last_frame_pushed_at": "2026-07-06T16:53:59+09:00", "last_frame_read_age_sec": 4.807, "last_frame_read_at": "2026-07-06T16:53:59+09:00", "mjpeg": {"age_sec": 4.666, "exists": true, "mtime": "2026-07-06T16:53:59.290514+09:00", "mtime_epoch": 1783324439.2905145, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197707}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:54:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1597.441, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 47, "pushed_frames": 47, "read_frames": 46, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.573, "pushed": 1.573, "read": 1.54}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.375, "count": 47, "max_ms": 9.509, "p95_ms": 8.135}, "gstreamer_push": {"avg_ms": 1.131, "count": 47, "max_ms": 1.271, "p95_ms": 1.241}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.278, "count": 47, "max_ms": 75.057, "p95_ms": 70.056}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.179, "count": 47, "max_ms": 111.575, "p95_ms": 103.626}, "yolo11_640_fire_11class": {"avg_ms": 61.309, "count": 47, "max_ms": 87.457, "p95_ms": 85.044}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.119, "count": 47, "max_ms": 95.051, "p95_ms": 92.057}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.096, "count": 47, "max_ms": 166.301, "p95_ms": 157.429}, "read_interval": {"avg_ms": 548.109, "count": 46, "max_ms": 18851.957, "p95_ms": 169.097}, "triton_wait": {"avg_ms": 77.192, "count": 47, "max_ms": 111.588, "p95_ms": 103.638}, "visualize": {"avg_ms": 5.185, "count": 47, "max_ms": 6.386, "p95_ms": 5.64}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5634.859, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56355fdcc240] error while decoding MB 95 52, bytestream -5 [in#0/rtsp @ 0x56355f259e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56355f259e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:55:25+09:00", "last_error_summary": "[h264 @ 0x56355fdcc240] error while decoding MB 95 52, bytestream -5 [in#0/rtsp @ 0x56355f259e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56355f259e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:40:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 875.529, "last_started_at": "2026-07-06T16:40:49+09:00", "pid": 17753, "process": {"exists": false, "pid": 17753}, "restart_count": 44, "restart_count_per_hour": 10, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.85, "sample_count": 2354, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 113683, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:55:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T16:55:24+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T16:55:24+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T16:55:24.967705+09:00", "mtime_epoch": 1783324524.9677055, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244511}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:55:25+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1349.559, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 91, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.04, "pushed": 3.04, "read": 3.04}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.069, "count": 91, "max_ms": 6.063, "p95_ms": 4.882}, "gstreamer_push": {"avg_ms": 1.63, "count": 91, "max_ms": 7.565, "p95_ms": 4.285}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.832, "count": 91, "max_ms": 78.344, "p95_ms": 69.971}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.17, "count": 91, "max_ms": 99.007, "p95_ms": 97.202}, "yolo11_640_fire_11class": {"avg_ms": 61.45, "count": 91, "max_ms": 85.927, "p95_ms": 81.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.204, "count": 91, "max_ms": 95.065, "p95_ms": 87.588}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.143, "count": 91, "max_ms": 111.668, "p95_ms": 109.829}, "read_interval": {"avg_ms": 329.178, "count": 91, "max_ms": 11307.51, "p95_ms": 123.771}, "triton_wait": {"avg_ms": 69.315, "count": 91, "max_ms": 99.02, "p95_ms": 97.213}, "visualize": {"avg_ms": 3.693, "count": 91, "max_ms": 4.369, "p95_ms": 4.163}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5639.438, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:55:25+09:00", "last_error_summary": "[h264 @ 0x56355fdcc240] error while decoding MB 95 52, bytestream -5 [in#0/rtsp @ 0x56355f259e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56355f259e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:55:25+09:00", "last_error_summary": "[h264 @ 0x56355fdcc240] error while decoding MB 95 52, bytestream -5 [in#0/rtsp @ 0x56355f259e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56355f259e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:55:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 875.566, "last_started_at": "2026-07-06T16:40:49+09:00", "pid": 17753, "process": {"exists": false, "pid": 17753}, "restart_count": 44, "restart_count_per_hour": 10, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.85, "sample_count": 2355, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 113684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:55:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T16:55:25+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-06T16:55:24+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-06T16:55:24.967705+09:00", "mtime_epoch": 1783324524.9677055, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244511}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:55:25+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1354.453, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.069, "pushed": 3.069, "read": 3.036}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.108, "count": 92, "max_ms": 7.674, "p95_ms": 4.926}, "gstreamer_push": {"avg_ms": 1.682, "count": 92, "max_ms": 7.565, "p95_ms": 5.748}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.81, "count": 92, "max_ms": 78.344, "p95_ms": 69.971}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.119, "count": 92, "max_ms": 99.007, "p95_ms": 97.202}, "yolo11_640_fire_11class": {"avg_ms": 61.446, "count": 92, "max_ms": 85.927, "p95_ms": 81.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.178, "count": 92, "max_ms": 95.065, "p95_ms": 87.588}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.127, "count": 92, "max_ms": 111.668, "p95_ms": 109.829}, "read_interval": {"avg_ms": 329.178, "count": 91, "max_ms": 11307.51, "p95_ms": 123.771}, "triton_wait": {"avg_ms": 69.263, "count": 92, "max_ms": 99.02, "p95_ms": 97.213}, "visualize": {"avg_ms": 3.691, "count": 92, "max_ms": 4.369, "p95_ms": 4.163}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5639.414, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:55:25+09:00", "last_error_summary": "[h264 @ 0x56355fdcc240] error while decoding MB 95 52, bytestream -5 [in#0/rtsp @ 0x56355f259e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56355f259e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:55:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 875.575, "last_started_at": "2026-07-06T16:40:49+09:00", "pid": 17753, "process": {"exists": false, "pid": 17753}, "restart_count": 44, "restart_count_per_hour": 10, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.85, "sample_count": 2355, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 113684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:55:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.02, "last_frame_pushed_at": "2026-07-06T16:55:25+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T16:55:24+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-06T16:55:24.967705+09:00", "mtime_epoch": 1783324524.9677055, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244511}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:55:25+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1354.453, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.068, "pushed": 3.068, "read": 3.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.108, "count": 92, "max_ms": 7.674, "p95_ms": 4.926}, "gstreamer_push": {"avg_ms": 1.682, "count": 92, "max_ms": 7.565, "p95_ms": 5.748}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.81, "count": 92, "max_ms": 78.344, "p95_ms": 69.971}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.119, "count": 92, "max_ms": 99.007, "p95_ms": 97.202}, "yolo11_640_fire_11class": {"avg_ms": 61.446, "count": 92, "max_ms": 85.927, "p95_ms": 81.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.178, "count": 92, "max_ms": 95.065, "p95_ms": 87.588}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.127, "count": 92, "max_ms": 111.668, "p95_ms": 109.829}, "read_interval": {"avg_ms": 329.178, "count": 91, "max_ms": 11307.51, "p95_ms": 123.771}, "triton_wait": {"avg_ms": 69.263, "count": 92, "max_ms": 99.02, "p95_ms": 97.213}, "visualize": {"avg_ms": 3.691, "count": 92, "max_ms": 4.369, "p95_ms": 4.163}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5653.055, "limit_mb": 8796093022207.996}, "detail": {"pid": 18440, "restart_count": 45, "start_count": 46}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:55:25+09:00", "last_error_summary": "[h264 @ 0x56355fdcc240] error while decoding MB 95 52, bytestream -5 [in#0/rtsp @ 0x56355f259e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56355f259e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:55:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:55:27+09:00", "pid": 18440, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.637, "name": "ffmpeg", "pid": 18440, "rss_mb": 58.637, "state": "S (sleeping)", "threads": 1}, "restart_count": 45, "restart_count_per_hour": 11, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.974, "sample_count": 2310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 113684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:55:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.551, "last_frame_pushed_at": "2026-07-06T16:55:25+09:00", "last_frame_read_age_sec": 4.64, "last_frame_read_at": "2026-07-06T16:55:24+09:00", "mjpeg": {"age_sec": 4.528, "exists": true, "mtime": "2026-07-06T16:55:25.067707+09:00", "mtime_epoch": 1783324525.0677068, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T16:55:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1594.508, "name": "celery", "pid": 267, "rss_mb": 1354.453, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 44, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.467, "pushed": 1.467, "read": 1.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.175, "count": 44, "max_ms": 7.674, "p95_ms": 5.091}, "gstreamer_push": {"avg_ms": 1.904, "count": 44, "max_ms": 6.379, "p95_ms": 5.867}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.264, "count": 44, "max_ms": 78.344, "p95_ms": 69.971}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.004, "count": 44, "max_ms": 97.73, "p95_ms": 97.202}, "yolo11_640_fire_11class": {"avg_ms": 62.706, "count": 44, "max_ms": 85.927, "p95_ms": 81.94}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.501, "count": 44, "max_ms": 95.065, "p95_ms": 88.466}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.03, "count": 44, "max_ms": 110.517, "p95_ms": 109.829}, "read_interval": {"avg_ms": 579.025, "count": 44, "max_ms": 11307.51, "p95_ms": 123.771}, "triton_wait": {"avg_ms": 69.16, "count": 44, "max_ms": 97.742, "p95_ms": 97.213}, "visualize": {"avg_ms": 3.673, "count": 44, "max_ms": 4.229, "p95_ms": 4.08}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5698.539, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T16:53:59+09:00", "last_error_summary": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:53:59+09:00", "last_error_summary": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "last_exit_at": "2026-07-06T16:57:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.584, "last_started_at": "2026-07-06T16:54:01+09:00", "pid": 18333, "process": {"exists": false, "pid": 18333}, "restart_count": 43, "restart_count_per_hour": 19, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.47, "sample_count": 2015, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 127437, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:57:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T16:57:17+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-06T16:57:17+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-06T16:57:17.389333+09:00", "mtime_epoch": 1783324637.389333, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192804}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:57:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1598.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 221, "pushed_frames": 221, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.394, "pushed": 7.394, "read": 7.36}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.669, "count": 221, "max_ms": 8.121, "p95_ms": 6.966}, "gstreamer_push": {"avg_ms": 1.172, "count": 221, "max_ms": 2.522, "p95_ms": 1.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.3, "count": 221, "max_ms": 86.282, "p95_ms": 77.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.728, "count": 221, "max_ms": 124.536, "p95_ms": 106.819}, "yolo11_640_fire_11class": {"avg_ms": 64.608, "count": 221, "max_ms": 93.816, "p95_ms": 85.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.038, "count": 221, "max_ms": 102.702, "p95_ms": 96.16}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.822, "count": 221, "max_ms": 174.601, "p95_ms": 157.745}, "read_interval": {"avg_ms": 135.859, "count": 220, "max_ms": 183.879, "p95_ms": 166.884}, "triton_wait": {"avg_ms": 79.874, "count": 221, "max_ms": 124.549, "p95_ms": 106.832}, "visualize": {"avg_ms": 4.9, "count": 221, "max_ms": 6.635, "p95_ms": 5.528}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5698.531, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:53:59+09:00", "last_error_summary": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "last_exit_at": "2026-07-06T16:57:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.589, "last_started_at": "2026-07-06T16:54:01+09:00", "pid": 18333, "process": {"exists": false, "pid": 18333}, "restart_count": 43, "restart_count_per_hour": 19, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.47, "sample_count": 2015, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 127437, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:57:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T16:57:17+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-06T16:57:17+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-06T16:57:17.389333+09:00", "mtime_epoch": 1783324637.389333, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192804}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:57:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1598.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 221, "pushed_frames": 221, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.392, "pushed": 7.392, "read": 7.359}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.669, "count": 221, "max_ms": 8.121, "p95_ms": 6.966}, "gstreamer_push": {"avg_ms": 1.172, "count": 221, "max_ms": 2.522, "p95_ms": 1.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.3, "count": 221, "max_ms": 86.282, "p95_ms": 77.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.728, "count": 221, "max_ms": 124.536, "p95_ms": 106.819}, "yolo11_640_fire_11class": {"avg_ms": 64.608, "count": 221, "max_ms": 93.816, "p95_ms": 85.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.038, "count": 221, "max_ms": 102.702, "p95_ms": 96.16}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.822, "count": 221, "max_ms": 174.601, "p95_ms": 157.745}, "read_interval": {"avg_ms": 135.859, "count": 220, "max_ms": 183.879, "p95_ms": 166.884}, "triton_wait": {"avg_ms": 79.874, "count": 221, "max_ms": 124.549, "p95_ms": 106.832}, "visualize": {"avg_ms": 4.9, "count": 221, "max_ms": 6.635, "p95_ms": 5.528}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5818.367, "limit_mb": 8796093022207.996}, "detail": {"pid": 18592, "restart_count": 44, "start_count": 45}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T16:53:59+09:00", "last_error_summary": "[h264 @ 0x564f5e29c240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5e29c240] error while decoding MB 0 33, bytestream 6301 [h264 @ 0x564f5ecfbe80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ecfbe80] error while decoding MB 0 34, bytestream 5363 [h264 @ 0x564f5ef68a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564f5ef68a00] error while decoding MB 0 5, bytestream 10952 [h264 @ 0x564f5e3c3400] left block unavailable for reque", "last_exit_at": "2026-07-06T16:57:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T16:57:20+09:00", "pid": 18592, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.254, "name": "ffmpeg", "pid": 18592, "rss_mb": 139.254, "state": "S (sleeping)", "threads": 106}, "restart_count": 44, "restart_count_per_hour": 20, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.541, "sample_count": 1979, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 127437, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T16:57:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.903, "last_frame_pushed_at": "2026-07-06T16:57:17+09:00", "last_frame_read_age_sec": 5.059, "last_frame_read_at": "2026-07-06T16:57:17+09:00", "mjpeg": {"age_sec": 4.888, "exists": true, "mtime": "2026-07-06T16:57:17.549335+09:00", "mtime_epoch": 1783324637.5493352, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193097}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T16:57:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1598.102, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 186, "pushed_frames": 186, "read_frames": 186, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.2, "pushed": 6.2, "read": 6.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.606, "count": 186, "max_ms": 8.121, "p95_ms": 6.966}, "gstreamer_push": {"avg_ms": 1.185, "count": 186, "max_ms": 2.522, "p95_ms": 1.412}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.274, "count": 186, "max_ms": 86.282, "p95_ms": 75.965}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.828, "count": 186, "max_ms": 116.658, "p95_ms": 106.778}, "yolo11_640_fire_11class": {"avg_ms": 64.786, "count": 186, "max_ms": 93.816, "p95_ms": 86.151}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.385, "count": 186, "max_ms": 102.702, "p95_ms": 95.555}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.748, "count": 186, "max_ms": 169.714, "p95_ms": 156.779}, "read_interval": {"avg_ms": 134.923, "count": 186, "max_ms": 178.544, "p95_ms": 166.144}, "triton_wait": {"avg_ms": 79.999, "count": 186, "max_ms": 116.671, "p95_ms": 106.791}, "visualize": {"avg_ms": 4.857, "count": 186, "max_ms": 6.635, "p95_ms": 5.528}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1072}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5717.746, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562d4607c180] error while decoding MB 12 28, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 195, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:00:25+09:00", "last_error_summary": "[h264 @ 0x562d4607c180] error while decoding MB 12 28, bytestream -5", "last_exit_at": "2026-07-06T16:57:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.915, "last_started_at": "2026-07-06T16:57:20+09:00", "pid": 18592, "process": {"exists": false, "pid": 18592}, "restart_count": 44, "restart_count_per_hour": 19, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.6, "sample_count": 2126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 128746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:00:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T17:00:25+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T17:00:25+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T17:00:25.304182+09:00", "mtime_epoch": 1783324825.3041818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196889}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:00:25+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1598.875, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 221, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.333, "pushed": 7.333, "read": 7.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.206, "count": 220, "max_ms": 9.26, "p95_ms": 7.244}, "gstreamer_push": {"avg_ms": 1.076, "count": 220, "max_ms": 1.351, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.068, "count": 220, "max_ms": 92.765, "p95_ms": 70.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.923, "count": 220, "max_ms": 113.309, "p95_ms": 103.095}, "yolo11_640_fire_11class": {"avg_ms": 57.397, "count": 220, "max_ms": 88.046, "p95_ms": 77.98}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.582, "count": 220, "max_ms": 111.524, "p95_ms": 95.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.519, "count": 220, "max_ms": 166.788, "p95_ms": 158.478}, "read_interval": {"avg_ms": 136.107, "count": 221, "max_ms": 178.966, "p95_ms": 167.332}, "triton_wait": {"avg_ms": 72.991, "count": 220, "max_ms": 113.322, "p95_ms": 103.107}, "visualize": {"avg_ms": 5.173, "count": 220, "max_ms": 6.707, "p95_ms": 5.828}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1072}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5717.312, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:00:25+09:00", "last_error_summary": "[h264 @ 0x562d4607c180] error while decoding MB 12 28, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 195, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:00:25+09:00", "last_error_summary": "[h264 @ 0x562d4607c180] error while decoding MB 12 28, bytestream -5", "last_exit_at": "2026-07-06T17:00:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.97, "last_started_at": "2026-07-06T16:57:20+09:00", "pid": 18592, "process": {"exists": false, "pid": 18592}, "restart_count": 44, "restart_count_per_hour": 19, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.6, "sample_count": 2126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 128747, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:00:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T17:00:25+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-06T17:00:25+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T17:00:25.304182+09:00", "mtime_epoch": 1783324825.3041818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196889}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:00:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1598.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 221, "pushed_frames": 221, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.382, "pushed": 7.382, "read": 7.348}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.199, "count": 221, "max_ms": 9.26, "p95_ms": 7.244}, "gstreamer_push": {"avg_ms": 1.076, "count": 221, "max_ms": 1.351, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.026, "count": 221, "max_ms": 92.765, "p95_ms": 70.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.875, "count": 221, "max_ms": 113.309, "p95_ms": 103.095}, "yolo11_640_fire_11class": {"avg_ms": 57.347, "count": 221, "max_ms": 88.046, "p95_ms": 77.98}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.561, "count": 221, "max_ms": 111.524, "p95_ms": 95.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.403, "count": 221, "max_ms": 166.788, "p95_ms": 158.478}, "read_interval": {"avg_ms": 136.129, "count": 220, "max_ms": 178.966, "p95_ms": 167.332}, "triton_wait": {"avg_ms": 72.942, "count": 221, "max_ms": 113.322, "p95_ms": 103.107}, "visualize": {"avg_ms": 5.166, "count": 221, "max_ms": 6.707, "p95_ms": 5.828}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1072}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5717.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 195, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:00:25+09:00", "last_error_summary": "[h264 @ 0x562d4607c180] error while decoding MB 12 28, bytestream -5", "last_exit_at": "2026-07-06T17:00:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.975, "last_started_at": "2026-07-06T16:57:20+09:00", "pid": 18592, "process": {"exists": false, "pid": 18592}, "restart_count": 44, "restart_count_per_hour": 19, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.6, "sample_count": 2126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 128747, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:00:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T17:00:25+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T17:00:25+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T17:00:25.304182+09:00", "mtime_epoch": 1783324825.3041818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196889}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:00:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1598.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 221, "pushed_frames": 221, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.381, "pushed": 7.381, "read": 7.347}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.199, "count": 221, "max_ms": 9.26, "p95_ms": 7.244}, "gstreamer_push": {"avg_ms": 1.076, "count": 221, "max_ms": 1.351, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.026, "count": 221, "max_ms": 92.765, "p95_ms": 70.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.875, "count": 221, "max_ms": 113.309, "p95_ms": 103.095}, "yolo11_640_fire_11class": {"avg_ms": 57.347, "count": 221, "max_ms": 88.046, "p95_ms": 77.98}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.561, "count": 221, "max_ms": 111.524, "p95_ms": 95.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.403, "count": 221, "max_ms": 166.788, "p95_ms": 158.478}, "read_interval": {"avg_ms": 136.129, "count": 220, "max_ms": 178.966, "p95_ms": 167.332}, "triton_wait": {"avg_ms": 72.942, "count": 221, "max_ms": 113.322, "p95_ms": 103.107}, "visualize": {"avg_ms": 5.166, "count": 221, "max_ms": 6.707, "p95_ms": 5.828}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1072}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5825.184, "limit_mb": 8796093022207.996}, "detail": {"pid": 18699, "restart_count": 45, "start_count": 46}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 195, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:00:25+09:00", "last_error_summary": "[h264 @ 0x562d4607c180] error while decoding MB 12 28, bytestream -5", "last_exit_at": "2026-07-06T17:00:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:00:28+09:00", "pid": 18699, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.078, "name": "ffmpeg", "pid": 18699, "rss_mb": 136.078, "state": "S (sleeping)", "threads": 106}, "restart_count": 45, "restart_count_per_hour": 20, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.712, "sample_count": 2086, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 128747, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:00:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.999, "last_frame_pushed_at": "2026-07-06T17:00:25+09:00", "last_frame_read_age_sec": 5.102, "last_frame_read_at": "2026-07-06T17:00:25+09:00", "mjpeg": {"age_sec": 4.99, "exists": true, "mtime": "2026-07-06T17:00:25.412184+09:00", "mtime_epoch": 1783324825.4121835, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196841}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:00:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1598.875, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.185, "pushed": 6.185, "read": 6.151}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.214, "count": 185, "max_ms": 9.26, "p95_ms": 7.333}, "gstreamer_push": {"avg_ms": 1.071, "count": 185, "max_ms": 1.351, "p95_ms": 1.203}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.413, "count": 185, "max_ms": 92.765, "p95_ms": 69.364}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.338, "count": 185, "max_ms": 113.309, "p95_ms": 102.355}, "yolo11_640_fire_11class": {"avg_ms": 56.633, "count": 185, "max_ms": 88.046, "p95_ms": 76.952}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.048, "count": 185, "max_ms": 111.524, "p95_ms": 93.678}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.852, "count": 185, "max_ms": 166.788, "p95_ms": 157.169}, "read_interval": {"avg_ms": 135.605, "count": 184, "max_ms": 178.966, "p95_ms": 166.485}, "triton_wait": {"avg_ms": 72.395, "count": 185, "max_ms": 113.322, "p95_ms": 102.367}, "visualize": {"avg_ms": 5.158, "count": 185, "max_ms": 6.707, "p95_ms": 5.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5870.387, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557db2deb640] error while decoding MB 6 24, bytestream -48 [in#0/rtsp @ 0x557db2d68e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db2d68e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:01:36+09:00", "last_error_summary": "[h264 @ 0x557db2deb640] error while decoding MB 6 24, bytestream -48 [in#0/rtsp @ 0x557db2d68e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db2d68e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T16:55:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 368.608, "last_started_at": "2026-07-06T16:55:27+09:00", "pid": 18440, "process": {"exists": false, "pid": 18440}, "restart_count": 45, "restart_count_per_hour": 10, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.111, "sample_count": 2504, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 116843, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:01:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-06T17:01:36+09:00", "last_frame_read_age_sec": 0.079, "last_frame_read_at": "2026-07-06T17:01:36+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-06T17:01:36.133288+09:00", "mtime_epoch": 1783324896.1332884, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 248037}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T17:01:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1577.887, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.537, "pushed": 3.537, "read": 3.503}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.305, "count": 106, "max_ms": 6.787, "p95_ms": 5.149}, "gstreamer_push": {"avg_ms": 1.292, "count": 106, "max_ms": 1.713, "p95_ms": 1.46}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.653, "count": 106, "max_ms": 84.043, "p95_ms": 75.765}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.924, "count": 106, "max_ms": 111.105, "p95_ms": 101.774}, "yolo11_640_fire_11class": {"avg_ms": 61.79, "count": 106, "max_ms": 87.502, "p95_ms": 82.1}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.597, "count": 106, "max_ms": 101.269, "p95_ms": 93.27}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.071, "count": 106, "max_ms": 123.923, "p95_ms": 116.056}, "read_interval": {"avg_ms": 285.436, "count": 105, "max_ms": 10704.452, "p95_ms": 130.41}, "triton_wait": {"avg_ms": 72.19, "count": 106, "max_ms": 111.117, "p95_ms": 101.79}, "visualize": {"avg_ms": 3.661, "count": 106, "max_ms": 5.523, "p95_ms": 4.392}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5867.531, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:01:36+09:00", "last_error_summary": "[h264 @ 0x557db2deb640] error while decoding MB 6 24, bytestream -48 [in#0/rtsp @ 0x557db2d68e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db2d68e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:01:36+09:00", "last_error_summary": "[h264 @ 0x557db2deb640] error while decoding MB 6 24, bytestream -48 [in#0/rtsp @ 0x557db2d68e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db2d68e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T17:01:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 368.634, "last_started_at": "2026-07-06T16:55:27+09:00", "pid": 18440, "process": {"exists": false, "pid": 18440}, "restart_count": 45, "restart_count_per_hour": 10, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.111, "sample_count": 2503, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 116843, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:01:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.027, "last_frame_pushed_at": "2026-07-06T17:01:36+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-06T17:01:36+09:00", "mjpeg": {"age_sec": 0.007, "exists": true, "mtime": "2026-07-06T17:01:36.221290+09:00", "mtime_epoch": 1783324896.2212896, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 248046}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T17:01:36+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1577.887, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.534, "pushed": 3.534, "read": 3.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.306, "count": 105, "max_ms": 6.787, "p95_ms": 5.149}, "gstreamer_push": {"avg_ms": 1.292, "count": 106, "max_ms": 1.713, "p95_ms": 1.46}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.591, "count": 105, "max_ms": 84.043, "p95_ms": 75.765}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.993, "count": 105, "max_ms": 111.105, "p95_ms": 101.774}, "yolo11_640_fire_11class": {"avg_ms": 61.784, "count": 105, "max_ms": 87.502, "p95_ms": 82.1}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.629, "count": 105, "max_ms": 101.269, "p95_ms": 93.27}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.071, "count": 106, "max_ms": 123.923, "p95_ms": 116.056}, "read_interval": {"avg_ms": 285.436, "count": 105, "max_ms": 10704.452, "p95_ms": 130.41}, "triton_wait": {"avg_ms": 72.262, "count": 105, "max_ms": 111.117, "p95_ms": 101.79}, "visualize": {"avg_ms": 3.661, "count": 105, "max_ms": 5.523, "p95_ms": 4.392}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5869.977, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:01:36+09:00", "last_error_summary": "[h264 @ 0x557db2deb640] error while decoding MB 6 24, bytestream -48 [in#0/rtsp @ 0x557db2d68e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db2d68e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T17:01:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 368.643, "last_started_at": "2026-07-06T16:55:27+09:00", "pid": 18440, "process": {"exists": false, "pid": 18440}, "restart_count": 45, "restart_count_per_hour": 10, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.111, "sample_count": 2503, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 116843, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:01:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.036, "last_frame_pushed_at": "2026-07-06T17:01:36+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-06T17:01:36+09:00", "mjpeg": {"age_sec": 0.016, "exists": true, "mtime": "2026-07-06T17:01:36.221290+09:00", "mtime_epoch": 1783324896.2212896, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 248046}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T17:01:36+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1577.887, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.501, "pushed": 3.501, "read": 3.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.306, "count": 105, "max_ms": 6.787, "p95_ms": 5.149}, "gstreamer_push": {"avg_ms": 1.292, "count": 105, "max_ms": 1.713, "p95_ms": 1.46}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.591, "count": 105, "max_ms": 84.043, "p95_ms": 75.765}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.993, "count": 105, "max_ms": 111.105, "p95_ms": 101.774}, "yolo11_640_fire_11class": {"avg_ms": 61.784, "count": 105, "max_ms": 87.502, "p95_ms": 82.1}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.629, "count": 105, "max_ms": 101.269, "p95_ms": 93.27}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.139, "count": 105, "max_ms": 123.923, "p95_ms": 116.056}, "read_interval": {"avg_ms": 285.436, "count": 105, "max_ms": 10704.452, "p95_ms": 130.41}, "triton_wait": {"avg_ms": 72.262, "count": 105, "max_ms": 111.117, "p95_ms": 101.79}, "visualize": {"avg_ms": 3.661, "count": 105, "max_ms": 5.523, "p95_ms": 4.392}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5993.488, "limit_mb": 8796093022207.996}, "detail": {"pid": 18866, "restart_count": 46, "start_count": 47}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:01:36+09:00", "last_error_summary": "[h264 @ 0x557db2deb640] error while decoding MB 6 24, bytestream -48 [in#0/rtsp @ 0x557db2d68e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db2d68e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T17:01:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:01:38+09:00", "pid": 18866, "process": {"exists": true, "fd_count": 5, "hwm_mb": 148.637, "name": "ffmpeg", "pid": 18866, "rss_mb": 148.637, "state": "R (running)", "threads": 49}, "restart_count": 46, "restart_count_per_hour": 11, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.599, "sample_count": 2456, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 116843, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:01:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.667, "last_frame_pushed_at": "2026-07-06T17:01:36+09:00", "last_frame_read_age_sec": 4.745, "last_frame_read_at": "2026-07-06T17:01:36+09:00", "mjpeg": {"age_sec": 4.647, "exists": true, "mtime": "2026-07-06T17:01:36.221290+09:00", "mtime_epoch": 1783324896.2212896, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 248046}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T17:01:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1577.887, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.968, "pushed": 1.968, "read": 1.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.27, "count": 59, "max_ms": 6.605, "p95_ms": 5.023}, "gstreamer_push": {"avg_ms": 1.293, "count": 59, "max_ms": 1.597, "p95_ms": 1.494}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.667, "count": 59, "max_ms": 84.043, "p95_ms": 77.746}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.543, "count": 59, "max_ms": 107.153, "p95_ms": 104.565}, "yolo11_640_fire_11class": {"avg_ms": 60.263, "count": 59, "max_ms": 82.1, "p95_ms": 79.565}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.344, "count": 59, "max_ms": 101.269, "p95_ms": 97.052}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.494, "count": 59, "max_ms": 120.008, "p95_ms": 117.537}, "read_interval": {"avg_ms": 436.738, "count": 58, "max_ms": 10704.452, "p95_ms": 139.16}, "triton_wait": {"avg_ms": 71.703, "count": 59, "max_ms": 107.165, "p95_ms": 104.578}, "visualize": {"avg_ms": 3.595, "count": 59, "max_ms": 4.694, "p95_ms": 4.302}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5840.836, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 10, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:02:41+09:00", "last_error_summary": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "last_exit_at": "2026-07-06T17:00:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.045, "last_started_at": "2026-07-06T17:00:28+09:00", "pid": 18699, "process": {"exists": false, "pid": 18699}, "restart_count": 45, "restart_count_per_hour": 20, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.874, "sample_count": 1998, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 129570, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T17:02:41+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T17:02:41+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T17:02:41.406321+09:00", "mtime_epoch": 1783324961.4063208, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194934}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:02:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1599.133, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 77, "pushed_frames": 77, "read_frames": 78, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.567, "pushed": 2.567, "read": 2.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.897, "count": 77, "max_ms": 7.338, "p95_ms": 7.01}, "gstreamer_push": {"avg_ms": 1.097, "count": 77, "max_ms": 1.619, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.932, "count": 77, "max_ms": 82.317, "p95_ms": 72.761}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.304, "count": 77, "max_ms": 110.006, "p95_ms": 107.208}, "yolo11_640_fire_11class": {"avg_ms": 59.376, "count": 77, "max_ms": 84.646, "p95_ms": 81.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.701, "count": 77, "max_ms": 98.651, "p95_ms": 94.749}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.264, "count": 77, "max_ms": 163.731, "p95_ms": 160.27}, "read_interval": {"avg_ms": 385.449, "count": 78, "max_ms": 10130.666, "p95_ms": 172.625}, "triton_wait": {"avg_ms": 75.482, "count": 77, "max_ms": 110.022, "p95_ms": 107.221}, "visualize": {"avg_ms": 5.038, "count": 77, "max_ms": 6.393, "p95_ms": 5.694}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5838.0, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:02:41+09:00", "last_error_summary": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:02:41+09:00", "last_error_summary": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "last_exit_at": "2026-07-06T17:02:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.121, "last_started_at": "2026-07-06T17:00:28+09:00", "pid": 18699, "process": {"exists": false, "pid": 18699}, "restart_count": 45, "restart_count_per_hour": 20, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.874, "sample_count": 1999, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 129571, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T17:02:41+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T17:02:41+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T17:02:41.406321+09:00", "mtime_epoch": 1783324961.4063208, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194934}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:02:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1599.133, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 77, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.604, "pushed": 2.604, "read": 2.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.902, "count": 78, "max_ms": 7.338, "p95_ms": 7.01}, "gstreamer_push": {"avg_ms": 1.096, "count": 78, "max_ms": 1.619, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.826, "count": 78, "max_ms": 82.317, "p95_ms": 72.761}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.2, "count": 78, "max_ms": 110.006, "p95_ms": 107.208}, "yolo11_640_fire_11class": {"avg_ms": 59.314, "count": 78, "max_ms": 84.646, "p95_ms": 81.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.643, "count": 78, "max_ms": 98.651, "p95_ms": 94.749}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.161, "count": 78, "max_ms": 163.731, "p95_ms": 160.27}, "read_interval": {"avg_ms": 388.841, "count": 77, "max_ms": 10130.666, "p95_ms": 172.625}, "triton_wait": {"avg_ms": 75.376, "count": 78, "max_ms": 110.022, "p95_ms": 107.221}, "visualize": {"avg_ms": 5.038, "count": 78, "max_ms": 6.393, "p95_ms": 5.694}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5837.977, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:02:41+09:00", "last_error_summary": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "last_exit_at": "2026-07-06T17:02:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.126, "last_started_at": "2026-07-06T17:00:28+09:00", "pid": 18699, "process": {"exists": false, "pid": 18699}, "restart_count": 45, "restart_count_per_hour": 20, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.874, "sample_count": 1999, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 129571, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T17:02:41+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T17:02:41+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T17:02:41.406321+09:00", "mtime_epoch": 1783324961.4063208, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194934}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:02:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1599.133, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 77, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.603, "pushed": 2.603, "read": 2.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.903, "count": 77, "max_ms": 7.338, "p95_ms": 7.01}, "gstreamer_push": {"avg_ms": 1.096, "count": 78, "max_ms": 1.619, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.993, "count": 77, "max_ms": 82.317, "p95_ms": 72.761}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.445, "count": 77, "max_ms": 110.006, "p95_ms": 107.208}, "yolo11_640_fire_11class": {"avg_ms": 59.455, "count": 77, "max_ms": 84.646, "p95_ms": 81.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.745, "count": 77, "max_ms": 98.651, "p95_ms": 94.749}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.161, "count": 78, "max_ms": 163.731, "p95_ms": 160.27}, "read_interval": {"avg_ms": 388.841, "count": 77, "max_ms": 10130.666, "p95_ms": 172.625}, "triton_wait": {"avg_ms": 75.592, "count": 77, "max_ms": 110.022, "p95_ms": 107.221}, "visualize": {"avg_ms": 5.034, "count": 77, "max_ms": 6.393, "p95_ms": 5.694}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5929.965, "limit_mb": 8796093022207.996}, "detail": {"pid": 19018, "restart_count": 46, "start_count": 47}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:02:41+09:00", "last_error_summary": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "last_exit_at": "2026-07-06T17:02:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:02:43+09:00", "pid": 19018, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.871, "name": "ffmpeg", "pid": 19018, "rss_mb": 138.871, "state": "S (sleeping)", "threads": 106}, "restart_count": 46, "restart_count_per_hour": 20, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.874, "sample_count": 1968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 129571, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.304, "last_frame_pushed_at": "2026-07-06T17:02:41+09:00", "last_frame_read_age_sec": 4.424, "last_frame_read_at": "2026-07-06T17:02:41+09:00", "mjpeg": {"age_sec": 4.284, "exists": true, "mtime": "2026-07-06T17:02:41.534323+09:00", "mtime_epoch": 1783324961.534323, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194677}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:02:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1599.133, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 47, "pushed_frames": 47, "read_frames": 46, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.567, "pushed": 1.567, "read": 1.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.917, "count": 46, "max_ms": 7.338, "p95_ms": 7.089}, "gstreamer_push": {"avg_ms": 1.075, "count": 47, "max_ms": 1.229, "p95_ms": 1.206}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.54, "count": 46, "max_ms": 82.317, "p95_ms": 71.382}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.501, "count": 46, "max_ms": 110.006, "p95_ms": 107.208}, "yolo11_640_fire_11class": {"avg_ms": 59.145, "count": 46, "max_ms": 84.646, "p95_ms": 81.834}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.549, "count": 46, "max_ms": 98.651, "p95_ms": 95.098}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.027, "count": 47, "max_ms": 161.584, "p95_ms": 160.27}, "read_interval": {"avg_ms": 558.466, "count": 46, "max_ms": 10130.666, "p95_ms": 172.625}, "triton_wait": {"avg_ms": 75.74, "count": 46, "max_ms": 110.022, "p95_ms": 107.221}, "visualize": {"avg_ms": 4.982, "count": 46, "max_ms": 6.393, "p95_ms": 5.646}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5830.906, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:02:41+09:00", "last_error_summary": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:02:41+09:00", "last_error_summary": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "last_exit_at": "2026-07-06T17:05:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.934, "last_started_at": "2026-07-06T17:02:43+09:00", "pid": 19018, "process": {"exists": false, "pid": 19018}, "restart_count": 46, "restart_count_per_hour": 19, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.626, "sample_count": 2013, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 130927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:05:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T17:05:53+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T17:05:53+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T17:05:53.645419+09:00", "mtime_epoch": 1783325153.6454194, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193686}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:05:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1599.906, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.212, "pushed": 7.212, "read": 7.179}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.134, "count": 216, "max_ms": 8.376, "p95_ms": 7.277}, "gstreamer_push": {"avg_ms": 1.124, "count": 216, "max_ms": 1.816, "p95_ms": 1.285}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.595, "count": 216, "max_ms": 85.746, "p95_ms": 70.823}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.108, "count": 216, "max_ms": 116.204, "p95_ms": 102.735}, "yolo11_640_fire_11class": {"avg_ms": 59.453, "count": 216, "max_ms": 85.199, "p95_ms": 77.665}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.179, "count": 216, "max_ms": 104.582, "p95_ms": 90.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.668, "count": 216, "max_ms": 171.039, "p95_ms": 156.148}, "read_interval": {"avg_ms": 139.291, "count": 215, "max_ms": 181.023, "p95_ms": 165.339}, "triton_wait": {"avg_ms": 76.146, "count": 216, "max_ms": 116.217, "p95_ms": 102.749}, "visualize": {"avg_ms": 5.218, "count": 216, "max_ms": 7.005, "p95_ms": 6.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5830.898, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:02:41+09:00", "last_error_summary": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "last_exit_at": "2026-07-06T17:05:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.94, "last_started_at": "2026-07-06T17:02:43+09:00", "pid": 19018, "process": {"exists": false, "pid": 19018}, "restart_count": 46, "restart_count_per_hour": 19, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.626, "sample_count": 2013, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 130927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:05:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T17:05:53+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-06T17:05:53+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T17:05:53.645419+09:00", "mtime_epoch": 1783325153.6454194, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193686}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:05:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1599.906, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.211, "pushed": 7.211, "read": 7.177}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.134, "count": 216, "max_ms": 8.376, "p95_ms": 7.277}, "gstreamer_push": {"avg_ms": 1.124, "count": 216, "max_ms": 1.816, "p95_ms": 1.285}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.595, "count": 216, "max_ms": 85.746, "p95_ms": 70.823}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.108, "count": 216, "max_ms": 116.204, "p95_ms": 102.735}, "yolo11_640_fire_11class": {"avg_ms": 59.453, "count": 216, "max_ms": 85.199, "p95_ms": 77.665}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.179, "count": 216, "max_ms": 104.582, "p95_ms": 90.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.668, "count": 216, "max_ms": 171.039, "p95_ms": 156.148}, "read_interval": {"avg_ms": 139.291, "count": 215, "max_ms": 181.023, "p95_ms": 165.339}, "triton_wait": {"avg_ms": 76.146, "count": 216, "max_ms": 116.217, "p95_ms": 102.749}, "visualize": {"avg_ms": 5.218, "count": 216, "max_ms": 7.005, "p95_ms": 6.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5942.887, "limit_mb": 8796093022207.996}, "detail": {"pid": 19125, "restart_count": 47, "start_count": 48}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:02:41+09:00", "last_error_summary": "[h264 @ 0x55be10125240] cabac decode of qscale diff failed at 21 41 [h264 @ 0x55be10125240] error while decoding MB 21 41, bytestream 90537 [h264 @ 0x55be10259740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be10259740] error while decoding MB 0 33, bytestream 1581 [h264 @ 0x55be10281ac0] cabac decode of qscale diff failed at 73 2 [h264 @ 0x55be10281ac0] error while decoding MB 73 2, bytestream 6099 [h264 @ 0x55be101910c0] error while decoding MB 11 41, bytestream -5 [h264 ", "last_exit_at": "2026-07-06T17:05:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:05:56+09:00", "pid": 19125, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.707, "name": "ffmpeg", "pid": 19125, "rss_mb": 141.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 47, "restart_count_per_hour": 20, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.626, "sample_count": 1980, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 130927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:05:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.491, "last_frame_pushed_at": "2026-07-06T17:05:53+09:00", "last_frame_read_age_sec": 4.604, "last_frame_read_at": "2026-07-06T17:05:53+09:00", "mjpeg": {"age_sec": 4.476, "exists": true, "mtime": "2026-07-06T17:05:53.761421+09:00", "mtime_epoch": 1783325153.7614212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193878}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:05:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1599.906, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.145, "pushed": 6.145, "read": 6.112}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.14, "count": 184, "max_ms": 8.376, "p95_ms": 7.236}, "gstreamer_push": {"avg_ms": 1.129, "count": 184, "max_ms": 1.816, "p95_ms": 1.29}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.555, "count": 184, "max_ms": 85.746, "p95_ms": 71.561}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.798, "count": 184, "max_ms": 116.204, "p95_ms": 102.735}, "yolo11_640_fire_11class": {"avg_ms": 59.226, "count": 184, "max_ms": 83.449, "p95_ms": 77.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.801, "count": 184, "max_ms": 97.483, "p95_ms": 89.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.51, "count": 184, "max_ms": 171.039, "p95_ms": 156.148}, "read_interval": {"avg_ms": 139.172, "count": 183, "max_ms": 180.821, "p95_ms": 165.339}, "triton_wait": {"avg_ms": 75.838, "count": 184, "max_ms": 116.217, "p95_ms": 102.749}, "visualize": {"avg_ms": 5.223, "count": 184, "max_ms": 7.005, "p95_ms": 6.097}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5839.57, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556de6ee8c00] error while decoding MB 68 22, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 206, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:09:05+09:00", "last_error_summary": "[h264 @ 0x556de6ee8c00] error while decoding MB 68 22, bytestream -14", "last_exit_at": "2026-07-06T17:05:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.396, "last_started_at": "2026-07-06T17:05:56+09:00", "pid": 19125, "process": {"exists": false, "pid": 19125}, "restart_count": 47, "restart_count_per_hour": 19, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.899, "sample_count": 2124, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132271, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:09:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T17:09:05+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T17:09:05+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-06T17:09:05.584578+09:00", "mtime_epoch": 1783325345.5845778, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:09:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1601.969, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 211, "pushed_frames": 211, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.06, "pushed": 7.06, "read": 7.06}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.25, "count": 211, "max_ms": 10.13, "p95_ms": 7.635}, "gstreamer_push": {"avg_ms": 1.132, "count": 211, "max_ms": 1.945, "p95_ms": 1.258}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.509, "count": 211, "max_ms": 83.654, "p95_ms": 73.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.991, "count": 211, "max_ms": 122.436, "p95_ms": 106.035}, "yolo11_640_fire_11class": {"avg_ms": 60.142, "count": 211, "max_ms": 91.533, "p95_ms": 79.857}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.367, "count": 211, "max_ms": 101.828, "p95_ms": 93.95}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.271, "count": 211, "max_ms": 174.486, "p95_ms": 161.371}, "read_interval": {"avg_ms": 142.104, "count": 211, "max_ms": 186.64, "p95_ms": 171.285}, "triton_wait": {"avg_ms": 78.102, "count": 211, "max_ms": 122.449, "p95_ms": 106.048}, "visualize": {"avg_ms": 5.252, "count": 211, "max_ms": 8.015, "p95_ms": 6.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5838.82, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:09:05+09:00", "last_error_summary": "[h264 @ 0x556de6ee8c00] error while decoding MB 68 22, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 206, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:09:05+09:00", "last_error_summary": "[h264 @ 0x556de6ee8c00] error while decoding MB 68 22, bytestream -14", "last_exit_at": "2026-07-06T17:09:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.467, "last_started_at": "2026-07-06T17:05:56+09:00", "pid": 19125, "process": {"exists": false, "pid": 19125}, "restart_count": 47, "restart_count_per_hour": 19, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.899, "sample_count": 2125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:09:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T17:09:05+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-06T17:09:05+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T17:09:05.584578+09:00", "mtime_epoch": 1783325345.5845778, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:09:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1601.969, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.076, "pushed": 7.076, "read": 7.043}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.253, "count": 211, "max_ms": 10.13, "p95_ms": 7.635}, "gstreamer_push": {"avg_ms": 1.131, "count": 212, "max_ms": 1.945, "p95_ms": 1.258}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.441, "count": 211, "max_ms": 83.654, "p95_ms": 73.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.871, "count": 211, "max_ms": 122.436, "p95_ms": 106.035}, "yolo11_640_fire_11class": {"avg_ms": 60.06, "count": 211, "max_ms": 91.533, "p95_ms": 79.857}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.25, "count": 211, "max_ms": 101.828, "p95_ms": 93.95}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.208, "count": 212, "max_ms": 174.486, "p95_ms": 161.371}, "read_interval": {"avg_ms": 142.104, "count": 211, "max_ms": 186.64, "p95_ms": 171.285}, "triton_wait": {"avg_ms": 77.982, "count": 211, "max_ms": 122.449, "p95_ms": 106.048}, "visualize": {"avg_ms": 5.252, "count": 211, "max_ms": 8.015, "p95_ms": 6.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5838.938, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 206, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:09:05+09:00", "last_error_summary": "[h264 @ 0x556de6ee8c00] error while decoding MB 68 22, bytestream -14", "last_exit_at": "2026-07-06T17:09:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.474, "last_started_at": "2026-07-06T17:05:56+09:00", "pid": 19125, "process": {"exists": false, "pid": 19125}, "restart_count": 47, "restart_count_per_hour": 19, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.899, "sample_count": 2125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:09:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T17:09:05+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T17:09:05+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-06T17:09:05.584578+09:00", "mtime_epoch": 1783325345.5845778, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:09:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1601.969, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.075, "pushed": 7.075, "read": 7.041}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.253, "count": 211, "max_ms": 10.13, "p95_ms": 7.635}, "gstreamer_push": {"avg_ms": 1.131, "count": 212, "max_ms": 1.945, "p95_ms": 1.258}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.441, "count": 211, "max_ms": 83.654, "p95_ms": 73.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.871, "count": 211, "max_ms": 122.436, "p95_ms": 106.035}, "yolo11_640_fire_11class": {"avg_ms": 60.06, "count": 211, "max_ms": 91.533, "p95_ms": 79.857}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.25, "count": 211, "max_ms": 101.828, "p95_ms": 93.95}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.208, "count": 212, "max_ms": 174.486, "p95_ms": 161.371}, "read_interval": {"avg_ms": 142.104, "count": 211, "max_ms": 186.64, "p95_ms": 171.285}, "triton_wait": {"avg_ms": 77.982, "count": 211, "max_ms": 122.449, "p95_ms": 106.048}, "visualize": {"avg_ms": 5.252, "count": 211, "max_ms": 8.015, "p95_ms": 6.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5944.566, "limit_mb": 8796093022207.996}, "detail": {"pid": 19232, "restart_count": 48, "start_count": 49}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 206, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:09:05+09:00", "last_error_summary": "[h264 @ 0x556de6ee8c00] error while decoding MB 68 22, bytestream -14", "last_exit_at": "2026-07-06T17:09:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:09:08+09:00", "pid": 19232, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.789, "name": "ffmpeg", "pid": 19232, "rss_mb": 136.789, "state": "S (sleeping)", "threads": 106}, "restart_count": 48, "restart_count_per_hour": 20, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.908, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:09:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.583, "last_frame_pushed_at": "2026-07-06T17:09:05+09:00", "last_frame_read_age_sec": 4.704, "last_frame_read_at": "2026-07-06T17:09:05+09:00", "mjpeg": {"age_sec": 4.568, "exists": true, "mtime": "2026-07-06T17:09:05.712580+09:00", "mtime_epoch": 1783325345.7125797, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:09:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1601.969, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.951, "pushed": 5.951, "read": 5.917}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.264, "count": 178, "max_ms": 10.13, "p95_ms": 7.845}, "gstreamer_push": {"avg_ms": 1.133, "count": 178, "max_ms": 1.945, "p95_ms": 1.258}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.061, "count": 178, "max_ms": 83.654, "p95_ms": 73.412}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.781, "count": 178, "max_ms": 122.436, "p95_ms": 109.65}, "yolo11_640_fire_11class": {"avg_ms": 60.767, "count": 178, "max_ms": 91.533, "p95_ms": 80.385}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.022, "count": 178, "max_ms": 101.828, "p95_ms": 93.987}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.13, "count": 178, "max_ms": 174.486, "p95_ms": 162.548}, "read_interval": {"avg_ms": 143.074, "count": 177, "max_ms": 186.64, "p95_ms": 174.119}, "triton_wait": {"avg_ms": 78.91, "count": 178, "max_ms": 122.449, "p95_ms": 109.661}, "visualize": {"avg_ms": 5.264, "count": 178, "max_ms": 8.015, "p95_ms": 6.227}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5841.414, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ce13bfd000] error while decoding MB 25 33, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:12:16+09:00", "last_error_summary": "[h264 @ 0x55ce13bfd000] error while decoding MB 25 33, bytestream -5", "last_exit_at": "2026-07-06T17:09:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 188.163, "last_started_at": "2026-07-06T17:09:08+09:00", "pid": 19232, "process": {"exists": false, "pid": 19232}, "restart_count": 48, "restart_count_per_hour": 19, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.321, "sample_count": 2120, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 133606, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:12:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T17:12:16+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T17:12:16+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T17:12:16.399761+09:00", "mtime_epoch": 1783325536.3997614, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188735}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:12:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1602.227, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.191, "pushed": 7.191, "read": 7.191}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.195, "count": 215, "max_ms": 10.517, "p95_ms": 7.857}, "gstreamer_push": {"avg_ms": 1.114, "count": 215, "max_ms": 1.663, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.974, "count": 215, "max_ms": 85.93, "p95_ms": 74.463}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.256, "count": 215, "max_ms": 113.051, "p95_ms": 104.336}, "yolo11_640_fire_11class": {"avg_ms": 60.887, "count": 215, "max_ms": 86.401, "p95_ms": 80.291}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.744, "count": 215, "max_ms": 101.862, "p95_ms": 93.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.808, "count": 215, "max_ms": 173.614, "p95_ms": 158.85}, "read_interval": {"avg_ms": 139.371, "count": 215, "max_ms": 184.471, "p95_ms": 170.04}, "triton_wait": {"avg_ms": 76.304, "count": 215, "max_ms": 113.065, "p95_ms": 104.348}, "visualize": {"avg_ms": 5.185, "count": 215, "max_ms": 8.933, "p95_ms": 6.024}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5840.531, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:12:16+09:00", "last_error_summary": "[h264 @ 0x55ce13bfd000] error while decoding MB 25 33, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:12:16+09:00", "last_error_summary": "[h264 @ 0x55ce13bfd000] error while decoding MB 25 33, bytestream -5", "last_exit_at": "2026-07-06T17:12:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 188.238, "last_started_at": "2026-07-06T17:09:08+09:00", "pid": 19232, "process": {"exists": false, "pid": 19232}, "restart_count": 48, "restart_count_per_hour": 19, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.321, "sample_count": 2121, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 133607, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:12:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T17:12:16+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-06T17:12:16+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-06T17:12:16.399761+09:00", "mtime_epoch": 1783325536.3997614, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188735}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:12:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1602.227, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.206, "pushed": 7.206, "read": 7.173}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.192, "count": 215, "max_ms": 10.517, "p95_ms": 7.857}, "gstreamer_push": {"avg_ms": 1.114, "count": 216, "max_ms": 1.663, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.974, "count": 215, "max_ms": 85.93, "p95_ms": 74.463}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.268, "count": 215, "max_ms": 113.051, "p95_ms": 104.336}, "yolo11_640_fire_11class": {"avg_ms": 60.892, "count": 215, "max_ms": 86.401, "p95_ms": 80.291}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.748, "count": 215, "max_ms": 101.862, "p95_ms": 93.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.765, "count": 216, "max_ms": 173.614, "p95_ms": 158.85}, "read_interval": {"avg_ms": 139.371, "count": 215, "max_ms": 184.471, "p95_ms": 170.04}, "triton_wait": {"avg_ms": 76.316, "count": 215, "max_ms": 113.065, "p95_ms": 104.348}, "visualize": {"avg_ms": 5.187, "count": 215, "max_ms": 8.933, "p95_ms": 6.024}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5840.52, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:12:16+09:00", "last_error_summary": "[h264 @ 0x55ce13bfd000] error while decoding MB 25 33, bytestream -5", "last_exit_at": "2026-07-06T17:12:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 188.244, "last_started_at": "2026-07-06T17:09:08+09:00", "pid": 19232, "process": {"exists": false, "pid": 19232}, "restart_count": 48, "restart_count_per_hour": 19, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.321, "sample_count": 2121, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 133607, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:12:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T17:12:16+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T17:12:16+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-06T17:12:16.399761+09:00", "mtime_epoch": 1783325536.3997614, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188735}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:12:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1602.227, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.205, "pushed": 7.205, "read": 7.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.192, "count": 215, "max_ms": 10.517, "p95_ms": 7.857}, "gstreamer_push": {"avg_ms": 1.114, "count": 216, "max_ms": 1.663, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.974, "count": 215, "max_ms": 85.93, "p95_ms": 74.463}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.268, "count": 215, "max_ms": 113.051, "p95_ms": 104.336}, "yolo11_640_fire_11class": {"avg_ms": 60.892, "count": 215, "max_ms": 86.401, "p95_ms": 80.291}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.748, "count": 215, "max_ms": 101.862, "p95_ms": 93.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.765, "count": 216, "max_ms": 173.614, "p95_ms": 158.85}, "read_interval": {"avg_ms": 139.371, "count": 215, "max_ms": 184.471, "p95_ms": 170.04}, "triton_wait": {"avg_ms": 76.316, "count": 215, "max_ms": 113.065, "p95_ms": 104.348}, "visualize": {"avg_ms": 5.187, "count": 215, "max_ms": 8.933, "p95_ms": 6.024}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5949.914, "limit_mb": 8796093022207.996}, "detail": {"pid": 19339, "restart_count": 49, "start_count": 50}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:12:16+09:00", "last_error_summary": "[h264 @ 0x55ce13bfd000] error while decoding MB 25 33, bytestream -5", "last_exit_at": "2026-07-06T17:12:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:12:19+09:00", "pid": 19339, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.344, "name": "ffmpeg", "pid": 19339, "rss_mb": 139.344, "state": "S (sleeping)", "threads": 106}, "restart_count": 49, "restart_count_per_hour": 20, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.321, "sample_count": 2085, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 133607, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:12:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.911, "last_frame_pushed_at": "2026-07-06T17:12:16+09:00", "last_frame_read_age_sec": 5.034, "last_frame_read_at": "2026-07-06T17:12:16+09:00", "mjpeg": {"age_sec": 4.893, "exists": true, "mtime": "2026-07-06T17:12:16.531764+09:00", "mtime_epoch": 1783325536.5317636, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189709}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:12:21+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1602.227, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.036, "pushed": 6.036, "read": 6.003}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.264, "count": 180, "max_ms": 10.517, "p95_ms": 7.857}, "gstreamer_push": {"avg_ms": 1.115, "count": 181, "max_ms": 1.663, "p95_ms": 1.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.881, "count": 180, "max_ms": 85.93, "p95_ms": 72.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.897, "count": 180, "max_ms": 111.147, "p95_ms": 104.336}, "yolo11_640_fire_11class": {"avg_ms": 60.467, "count": 180, "max_ms": 86.401, "p95_ms": 79.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.111, "count": 180, "max_ms": 101.862, "p95_ms": 91.773}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.867, "count": 181, "max_ms": 173.614, "p95_ms": 158.85}, "read_interval": {"avg_ms": 139.507, "count": 180, "max_ms": 184.471, "p95_ms": 170.04}, "triton_wait": {"avg_ms": 75.945, "count": 180, "max_ms": 111.16, "p95_ms": 104.348}, "visualize": {"avg_ms": 5.229, "count": 180, "max_ms": 8.933, "p95_ms": 6.086}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5845.059, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55aded265bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aded265bc0] error while decoding MB 0 30, bytestream 44125 [h264 @ 0x55adec783380] cabac decode of qscale diff failed at 71 27 [h264 @ 0x55adec783380] error while decoding MB 71 27, bytestream 5970 [in#0/rtsp @ 0x55adec636e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55adec636e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 210, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:15:07+09:00", "last_error_summary": "[h264 @ 0x55aded265bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aded265bc0] error while decoding MB 0 30, bytestream 44125 [h264 @ 0x55adec783380] cabac decode of qscale diff failed at 71 27 [h264 @ 0x55adec783380] error while decoding MB 71 27, bytestream 5970 [in#0/rtsp @ 0x55adec636e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55adec636e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T17:12:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.937, "last_started_at": "2026-07-06T17:12:19+09:00", "pid": 19339, "process": {"exists": false, "pid": 19339}, "restart_count": 49, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.712, "sample_count": 1966, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 134647, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:15:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.089, "last_frame_pushed_at": "2026-07-06T17:15:07+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-06T17:15:07+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-06T17:15:07.290639+09:00", "mtime_epoch": 1783325707.2906387, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203098}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:15:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1602.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 71, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.372, "pushed": 2.372, "read": 2.372}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.063, "count": 72, "max_ms": 8.78, "p95_ms": 7.746}, "gstreamer_push": {"avg_ms": 1.101, "count": 71, "max_ms": 1.551, "p95_ms": 1.233}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.833, "count": 72, "max_ms": 83.587, "p95_ms": 78.377}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.605, "count": 72, "max_ms": 124.553, "p95_ms": 103.6}, "yolo11_640_fire_11class": {"avg_ms": 60.178, "count": 72, "max_ms": 83.988, "p95_ms": 80.964}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.819, "count": 72, "max_ms": 99.55, "p95_ms": 95.444}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.025, "count": 71, "max_ms": 180.113, "p95_ms": 155.463}, "read_interval": {"avg_ms": 422.436, "count": 71, "max_ms": 10241.707, "p95_ms": 180.95}, "triton_wait": {"avg_ms": 76.762, "count": 72, "max_ms": 124.567, "p95_ms": 103.613}, "visualize": {"avg_ms": 5.118, "count": 72, "max_ms": 6.727, "p95_ms": 6.372}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5844.809, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:15:07+09:00", "last_error_summary": "[h264 @ 0x55aded265bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aded265bc0] error while decoding MB 0 30, bytestream 44125 [h264 @ 0x55adec783380] cabac decode of qscale diff failed at 71 27 [h264 @ 0x55adec783380] error while decoding MB 71 27, bytestream 5970 [in#0/rtsp @ 0x55adec636e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55adec636e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 210, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:15:07+09:00", "last_error_summary": "[h264 @ 0x55aded265bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aded265bc0] error while decoding MB 0 30, bytestream 44125 [h264 @ 0x55adec783380] cabac decode of qscale diff failed at 71 27 [h264 @ 0x55adec783380] error while decoding MB 71 27, bytestream 5970 [in#0/rtsp @ 0x55adec636e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55adec636e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T17:15:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.981, "last_started_at": "2026-07-06T17:12:19+09:00", "pid": 19339, "process": {"exists": false, "pid": 19339}, "restart_count": 49, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.712, "sample_count": 1967, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 134647, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:15:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T17:15:07+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T17:15:07+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T17:15:07.290639+09:00", "mtime_epoch": 1783325707.2906387, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203098}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:15:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1602.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.402, "pushed": 2.402, "read": 2.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.067, "count": 71, "max_ms": 8.78, "p95_ms": 7.746}, "gstreamer_push": {"avg_ms": 1.104, "count": 72, "max_ms": 1.551, "p95_ms": 1.278}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.696, "count": 71, "max_ms": 83.587, "p95_ms": 78.377}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.334, "count": 71, "max_ms": 124.553, "p95_ms": 103.6}, "yolo11_640_fire_11class": {"avg_ms": 60.131, "count": 71, "max_ms": 83.988, "p95_ms": 80.964}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.811, "count": 71, "max_ms": 99.55, "p95_ms": 95.444}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.852, "count": 72, "max_ms": 180.113, "p95_ms": 155.463}, "read_interval": {"avg_ms": 422.436, "count": 71, "max_ms": 10241.707, "p95_ms": 180.95}, "triton_wait": {"avg_ms": 76.494, "count": 71, "max_ms": 124.567, "p95_ms": 103.613}, "visualize": {"avg_ms": 5.119, "count": 71, "max_ms": 6.727, "p95_ms": 6.372}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5844.777, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 210, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:15:07+09:00", "last_error_summary": "[h264 @ 0x55aded265bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aded265bc0] error while decoding MB 0 30, bytestream 44125 [h264 @ 0x55adec783380] cabac decode of qscale diff failed at 71 27 [h264 @ 0x55adec783380] error while decoding MB 71 27, bytestream 5970 [in#0/rtsp @ 0x55adec636e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55adec636e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T17:15:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.987, "last_started_at": "2026-07-06T17:12:19+09:00", "pid": 19339, "process": {"exists": false, "pid": 19339}, "restart_count": 49, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.712, "sample_count": 1967, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 134647, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:15:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T17:15:07+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T17:15:07+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T17:15:07.290639+09:00", "mtime_epoch": 1783325707.2906387, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203098}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:15:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1602.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.402, "pushed": 2.402, "read": 2.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.067, "count": 71, "max_ms": 8.78, "p95_ms": 7.746}, "gstreamer_push": {"avg_ms": 1.104, "count": 72, "max_ms": 1.551, "p95_ms": 1.278}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.696, "count": 71, "max_ms": 83.587, "p95_ms": 78.377}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.334, "count": 71, "max_ms": 124.553, "p95_ms": 103.6}, "yolo11_640_fire_11class": {"avg_ms": 60.131, "count": 71, "max_ms": 83.988, "p95_ms": 80.964}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.811, "count": 71, "max_ms": 99.55, "p95_ms": 95.444}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.852, "count": 72, "max_ms": 180.113, "p95_ms": 155.463}, "read_interval": {"avg_ms": 422.436, "count": 71, "max_ms": 10241.707, "p95_ms": 180.95}, "triton_wait": {"avg_ms": 76.494, "count": 71, "max_ms": 124.567, "p95_ms": 103.613}, "visualize": {"avg_ms": 5.119, "count": 71, "max_ms": 6.727, "p95_ms": 6.372}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5961.848, "limit_mb": 8796093022207.996}, "detail": {"pid": 19446, "restart_count": 50, "start_count": 51}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 210, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:15:07+09:00", "last_error_summary": "[h264 @ 0x55aded265bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55aded265bc0] error while decoding MB 0 30, bytestream 44125 [h264 @ 0x55adec783380] cabac decode of qscale diff failed at 71 27 [h264 @ 0x55adec783380] error while decoding MB 71 27, bytestream 5970 [in#0/rtsp @ 0x55adec636e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55adec636e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T17:15:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:15:10+09:00", "pid": 19446, "process": {"exists": true, "fd_count": 5, "hwm_mb": 143.973, "name": "ffmpeg", "pid": 19446, "rss_mb": 143.973, "state": "S (sleeping)", "threads": 106}, "restart_count": 50, "restart_count_per_hour": 20, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.696, "sample_count": 1933, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 134647, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:15:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.73, "last_frame_pushed_at": "2026-07-06T17:15:07+09:00", "last_frame_read_age_sec": 4.85, "last_frame_read_at": "2026-07-06T17:15:07+09:00", "mjpeg": {"age_sec": 4.711, "exists": true, "mtime": "2026-07-06T17:15:07.418641+09:00", "mtime_epoch": 1783325707.4186409, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202331}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:15:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1602.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.696, "pushed": 2.696, "read": 2.696}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.088, "count": 64, "max_ms": 8.78, "p95_ms": 7.746}, "gstreamer_push": {"avg_ms": 1.114, "count": 64, "max_ms": 1.551, "p95_ms": 1.278}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.435, "count": 64, "max_ms": 83.587, "p95_ms": 77.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.453, "count": 64, "max_ms": 124.553, "p95_ms": 101.166}, "yolo11_640_fire_11class": {"avg_ms": 60.843, "count": 64, "max_ms": 83.988, "p95_ms": 80.964}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.751, "count": 64, "max_ms": 99.55, "p95_ms": 95.444}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.877, "count": 64, "max_ms": 180.113, "p95_ms": 153.442}, "read_interval": {"avg_ms": 453.725, "count": 64, "max_ms": 10241.707, "p95_ms": 177.009}, "triton_wait": {"avg_ms": 76.629, "count": 64, "max_ms": 124.567, "p95_ms": 101.177}, "visualize": {"avg_ms": 5.132, "count": 64, "max_ms": 6.727, "p95_ms": 6.372}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.742, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5850.824, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556907a25d00] error while decoding MB 7 36, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 211, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:18:15+09:00", "last_error_summary": "[h264 @ 0x556907a25d00] error while decoding MB 7 36, bytestream -12", "last_exit_at": "2026-07-06T17:15:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.962, "last_started_at": "2026-07-06T17:15:10+09:00", "pid": 19446, "process": {"exists": false, "pid": 19446}, "restart_count": 50, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.977, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 135955, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:18:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T17:18:15+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T17:18:15+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T17:18:15.048619+09:00", "mtime_epoch": 1783325895.0486188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193266}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:18:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1603.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 212, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.083, "pushed": 7.083, "read": 7.083}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.307, "count": 212, "max_ms": 10.858, "p95_ms": 8.1}, "gstreamer_push": {"avg_ms": 1.132, "count": 212, "max_ms": 1.511, "p95_ms": 1.272}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.428, "count": 212, "max_ms": 81.166, "p95_ms": 73.815}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.198, "count": 212, "max_ms": 113.151, "p95_ms": 101.96}, "yolo11_640_fire_11class": {"avg_ms": 59.954, "count": 212, "max_ms": 88.989, "p95_ms": 79.201}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.264, "count": 212, "max_ms": 99.352, "p95_ms": 93.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.741, "count": 212, "max_ms": 168.304, "p95_ms": 157.214}, "read_interval": {"avg_ms": 141.436, "count": 212, "max_ms": 179.339, "p95_ms": 167.411}, "triton_wait": {"avg_ms": 77.326, "count": 212, "max_ms": 113.164, "p95_ms": 101.974}, "visualize": {"avg_ms": 5.302, "count": 212, "max_ms": 8.978, "p95_ms": 6.288}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5850.453, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:18:15+09:00", "last_error_summary": "[h264 @ 0x556907a25d00] error while decoding MB 7 36, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 211, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:18:15+09:00", "last_error_summary": "[h264 @ 0x556907a25d00] error while decoding MB 7 36, bytestream -12", "last_exit_at": "2026-07-06T17:18:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.059, "last_started_at": "2026-07-06T17:15:10+09:00", "pid": 19446, "process": {"exists": false, "pid": 19446}, "restart_count": 50, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.977, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 135956, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:18:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T17:18:15+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-06T17:18:15+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-06T17:18:15.048619+09:00", "mtime_epoch": 1783325895.0486188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193266}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:18:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1603.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.092, "pushed": 7.092, "read": 7.059}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.307, "count": 212, "max_ms": 10.858, "p95_ms": 8.1}, "gstreamer_push": {"avg_ms": 1.133, "count": 212, "max_ms": 1.511, "p95_ms": 1.272}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.546, "count": 212, "max_ms": 81.166, "p95_ms": 73.815}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.369, "count": 212, "max_ms": 113.151, "p95_ms": 101.96}, "yolo11_640_fire_11class": {"avg_ms": 60.076, "count": 212, "max_ms": 88.989, "p95_ms": 79.201}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.407, "count": 212, "max_ms": 99.352, "p95_ms": 93.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.912, "count": 212, "max_ms": 168.304, "p95_ms": 157.214}, "read_interval": {"avg_ms": 141.567, "count": 211, "max_ms": 179.339, "p95_ms": 167.411}, "triton_wait": {"avg_ms": 77.497, "count": 212, "max_ms": 113.164, "p95_ms": 101.974}, "visualize": {"avg_ms": 5.302, "count": 212, "max_ms": 8.978, "p95_ms": 6.288}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5850.445, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 211, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:18:15+09:00", "last_error_summary": "[h264 @ 0x556907a25d00] error while decoding MB 7 36, bytestream -12", "last_exit_at": "2026-07-06T17:18:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.065, "last_started_at": "2026-07-06T17:15:10+09:00", "pid": 19446, "process": {"exists": false, "pid": 19446}, "restart_count": 50, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.977, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 135956, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:18:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T17:18:15+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-06T17:18:15+09:00", "mjpeg": {"age_sec": 0.145, "exists": true, "mtime": "2026-07-06T17:18:15.048619+09:00", "mtime_epoch": 1783325895.0486188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193266}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:18:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1603.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.091, "pushed": 7.091, "read": 7.058}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.307, "count": 212, "max_ms": 10.858, "p95_ms": 8.1}, "gstreamer_push": {"avg_ms": 1.133, "count": 212, "max_ms": 1.511, "p95_ms": 1.272}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.546, "count": 212, "max_ms": 81.166, "p95_ms": 73.815}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.369, "count": 212, "max_ms": 113.151, "p95_ms": 101.96}, "yolo11_640_fire_11class": {"avg_ms": 60.076, "count": 212, "max_ms": 88.989, "p95_ms": 79.201}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.407, "count": 212, "max_ms": 99.352, "p95_ms": 93.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.912, "count": 212, "max_ms": 168.304, "p95_ms": 157.214}, "read_interval": {"avg_ms": 141.567, "count": 211, "max_ms": 179.339, "p95_ms": 167.411}, "triton_wait": {"avg_ms": 77.497, "count": 212, "max_ms": 113.164, "p95_ms": 101.974}, "visualize": {"avg_ms": 5.302, "count": 212, "max_ms": 8.978, "p95_ms": 6.288}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.301, "limit_mb": 8796093022207.996}, "detail": {"pid": 19553, "restart_count": 51, "start_count": 52}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 211, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:18:15+09:00", "last_error_summary": "[h264 @ 0x556907a25d00] error while decoding MB 7 36, bytestream -12", "last_exit_at": "2026-07-06T17:18:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:18:17+09:00", "pid": 19553, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.566, "name": "ffmpeg", "pid": 19553, "rss_mb": 136.566, "state": "R (running)", "threads": 106}, "restart_count": 51, "restart_count_per_hour": 20, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.934, "sample_count": 1936, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 135956, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:18:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.608, "last_frame_pushed_at": "2026-07-06T17:18:15+09:00", "last_frame_read_age_sec": 4.751, "last_frame_read_at": "2026-07-06T17:18:15+09:00", "mjpeg": {"age_sec": 4.594, "exists": true, "mtime": "2026-07-06T17:18:15.196620+09:00", "mtime_epoch": 1783325895.1966205, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193544}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:18:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1603.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.019, "pushed": 6.019, "read": 5.986}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.345, "count": 180, "max_ms": 10.858, "p95_ms": 8.215}, "gstreamer_push": {"avg_ms": 1.132, "count": 180, "max_ms": 1.511, "p95_ms": 1.271}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.394, "count": 180, "max_ms": 81.166, "p95_ms": 73.815}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.967, "count": 180, "max_ms": 113.151, "p95_ms": 101.96}, "yolo11_640_fire_11class": {"avg_ms": 59.64, "count": 180, "max_ms": 88.989, "p95_ms": 78.91}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.064, "count": 180, "max_ms": 99.352, "p95_ms": 93.038}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.566, "count": 180, "max_ms": 167.298, "p95_ms": 157.214}, "read_interval": {"avg_ms": 141.201, "count": 179, "max_ms": 179.339, "p95_ms": 167.411}, "triton_wait": {"avg_ms": 77.115, "count": 180, "max_ms": 113.164, "p95_ms": 101.974}, "visualize": {"avg_ms": 5.324, "count": 180, "max_ms": 8.978, "p95_ms": 6.355}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5848.828, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d650311b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d650311b80] error while decoding MB 0 37, bytestream 1814", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 212, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:21:22+09:00", "last_error_summary": "[h264 @ 0x55d650311b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d650311b80] error while decoding MB 0 37, bytestream 1814", "last_exit_at": "2026-07-06T17:18:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.972, "last_started_at": "2026-07-06T17:18:17+09:00", "pid": 19553, "process": {"exists": false, "pid": 19553}, "restart_count": 51, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.967, "sample_count": 2114, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 137269, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:21:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T17:21:22+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-06T17:21:22+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T17:21:22.710772+09:00", "mtime_epoch": 1783326082.7107718, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191712}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:21:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1603.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.233, "pushed": 7.233, "read": 7.233}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.127, "count": 216, "max_ms": 9.309, "p95_ms": 7.539}, "gstreamer_push": {"avg_ms": 1.117, "count": 216, "max_ms": 1.663, "p95_ms": 1.235}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.88, "count": 216, "max_ms": 88.937, "p95_ms": 73.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.217, "count": 216, "max_ms": 115.977, "p95_ms": 100.315}, "yolo11_640_fire_11class": {"avg_ms": 59.29, "count": 216, "max_ms": 91.11, "p95_ms": 75.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.48, "count": 216, "max_ms": 112.174, "p95_ms": 90.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.114, "count": 216, "max_ms": 176.011, "p95_ms": 152.951}, "read_interval": {"avg_ms": 138.663, "count": 216, "max_ms": 187.813, "p95_ms": 162.174}, "triton_wait": {"avg_ms": 75.472, "count": 216, "max_ms": 115.991, "p95_ms": 100.379}, "visualize": {"avg_ms": 5.168, "count": 216, "max_ms": 7.377, "p95_ms": 5.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.865, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5839.098, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:21:22+09:00", "last_error_summary": "[h264 @ 0x55d650311b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d650311b80] error while decoding MB 0 37, bytestream 1814", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 212, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:21:22+09:00", "last_error_summary": "[h264 @ 0x55d650311b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d650311b80] error while decoding MB 0 37, bytestream 1814", "last_exit_at": "2026-07-06T17:21:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.051, "last_started_at": "2026-07-06T17:18:17+09:00", "pid": 19553, "process": {"exists": false, "pid": 19553}, "restart_count": 51, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.967, "sample_count": 2115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 137270, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:21:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T17:21:22+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T17:21:22+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T17:21:22.710772+09:00", "mtime_epoch": 1783326082.7107718, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191712}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:21:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1603.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.247, "pushed": 7.247, "read": 7.214}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.136, "count": 217, "max_ms": 9.309, "p95_ms": 7.542}, "gstreamer_push": {"avg_ms": 1.116, "count": 217, "max_ms": 1.663, "p95_ms": 1.235}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.864, "count": 217, "max_ms": 88.937, "p95_ms": 73.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.189, "count": 217, "max_ms": 115.977, "p95_ms": 100.315}, "yolo11_640_fire_11class": {"avg_ms": 59.28, "count": 217, "max_ms": 91.11, "p95_ms": 75.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.469, "count": 217, "max_ms": 112.174, "p95_ms": 90.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.112, "count": 217, "max_ms": 176.011, "p95_ms": 152.951}, "read_interval": {"avg_ms": 138.663, "count": 216, "max_ms": 187.813, "p95_ms": 162.174}, "triton_wait": {"avg_ms": 75.442, "count": 217, "max_ms": 115.991, "p95_ms": 100.379}, "visualize": {"avg_ms": 5.17, "count": 217, "max_ms": 7.377, "p95_ms": 5.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5839.09, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 212, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:21:22+09:00", "last_error_summary": "[h264 @ 0x55d650311b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d650311b80] error while decoding MB 0 37, bytestream 1814", "last_exit_at": "2026-07-06T17:21:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.056, "last_started_at": "2026-07-06T17:18:17+09:00", "pid": 19553, "process": {"exists": false, "pid": 19553}, "restart_count": 51, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.967, "sample_count": 2115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 137270, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:21:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T17:21:22+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T17:21:22+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-06T17:21:22.710772+09:00", "mtime_epoch": 1783326082.7107718, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191712}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:21:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1603.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.246, "pushed": 7.246, "read": 7.212}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.136, "count": 217, "max_ms": 9.309, "p95_ms": 7.542}, "gstreamer_push": {"avg_ms": 1.116, "count": 217, "max_ms": 1.663, "p95_ms": 1.235}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.864, "count": 217, "max_ms": 88.937, "p95_ms": 73.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.189, "count": 217, "max_ms": 115.977, "p95_ms": 100.315}, "yolo11_640_fire_11class": {"avg_ms": 59.28, "count": 217, "max_ms": 91.11, "p95_ms": 75.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.469, "count": 217, "max_ms": 112.174, "p95_ms": 90.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.112, "count": 217, "max_ms": 176.011, "p95_ms": 152.951}, "read_interval": {"avg_ms": 138.663, "count": 216, "max_ms": 187.813, "p95_ms": 162.174}, "triton_wait": {"avg_ms": 75.442, "count": 217, "max_ms": 115.991, "p95_ms": 100.379}, "visualize": {"avg_ms": 5.17, "count": 217, "max_ms": 7.377, "p95_ms": 5.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.152, "limit_mb": 8796093022207.996}, "detail": {"pid": 19660, "restart_count": 52, "start_count": 53}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 212, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:21:22+09:00", "last_error_summary": "[h264 @ 0x55d650311b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d650311b80] error while decoding MB 0 37, bytestream 1814", "last_exit_at": "2026-07-06T17:21:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:21:25+09:00", "pid": 19660, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.645, "name": "ffmpeg", "pid": 19660, "rss_mb": 136.645, "state": "S (sleeping)", "threads": 106}, "restart_count": 52, "restart_count_per_hour": 20, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.07, "sample_count": 2083, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 137270, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:21:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.235, "last_frame_pushed_at": "2026-07-06T17:21:22+09:00", "last_frame_read_age_sec": 4.365, "last_frame_read_at": "2026-07-06T17:21:22+09:00", "mjpeg": {"age_sec": 4.216, "exists": true, "mtime": "2026-07-06T17:21:22.850774+09:00", "mtime_epoch": 1783326082.8507736, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191846}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:21:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1603.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 186, "pushed_frames": 186, "read_frames": 185, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.221, "pushed": 6.221, "read": 6.187}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.155, "count": 186, "max_ms": 9.309, "p95_ms": 7.543}, "gstreamer_push": {"avg_ms": 1.118, "count": 186, "max_ms": 1.663, "p95_ms": 1.235}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.491, "count": 186, "max_ms": 88.937, "p95_ms": 70.385}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.417, "count": 186, "max_ms": 115.977, "p95_ms": 98.433}, "yolo11_640_fire_11class": {"avg_ms": 58.995, "count": 186, "max_ms": 88.925, "p95_ms": 74.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.289, "count": 186, "max_ms": 102.499, "p95_ms": 89.926}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.287, "count": 186, "max_ms": 168.928, "p95_ms": 152.68}, "read_interval": {"avg_ms": 138.851, "count": 185, "max_ms": 177.981, "p95_ms": 161.779}, "triton_wait": {"avg_ms": 75.583, "count": 186, "max_ms": 115.991, "p95_ms": 98.447}, "visualize": {"avg_ms": 5.184, "count": 186, "max_ms": 7.377, "p95_ms": 5.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1155}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5850.82, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a27d044b40] cabac decode of qscale diff failed at 8 41 [h264 @ 0x55a27d044b40] error while decoding MB 8 41, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:24:34+09:00", "last_error_summary": "[h264 @ 0x55a27d044b40] cabac decode of qscale diff failed at 8 41 [h264 @ 0x55a27d044b40] error while decoding MB 8 41, bytestream -9", "last_exit_at": "2026-07-06T17:21:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 188.979, "last_started_at": "2026-07-06T17:21:25+09:00", "pid": 19660, "process": {"exists": false, "pid": 19660}, "restart_count": 52, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.886, "sample_count": 2123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 138612, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:24:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.053, "last_frame_pushed_at": "2026-07-06T17:24:33+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-06T17:24:33+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-06T17:24:34.009225+09:00", "mtime_epoch": 1783326274.0092251, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192405}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:24:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1604.547, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.135, "pushed": 7.135, "read": 7.168}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.131, "count": 214, "max_ms": 9.472, "p95_ms": 7.706}, "gstreamer_push": {"avg_ms": 1.116, "count": 214, "max_ms": 1.775, "p95_ms": 1.284}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.169, "count": 214, "max_ms": 86.319, "p95_ms": 76.233}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.746, "count": 214, "max_ms": 120.398, "p95_ms": 105.021}, "yolo11_640_fire_11class": {"avg_ms": 61.532, "count": 214, "max_ms": 88.489, "p95_ms": 80.598}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.225, "count": 214, "max_ms": 97.985, "p95_ms": 91.892}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.266, "count": 214, "max_ms": 180.689, "p95_ms": 162.758}, "read_interval": {"avg_ms": 139.904, "count": 215, "max_ms": 191.749, "p95_ms": 172.521}, "triton_wait": {"avg_ms": 77.102, "count": 214, "max_ms": 120.411, "p95_ms": 105.034}, "visualize": {"avg_ms": 5.112, "count": 214, "max_ms": 6.54, "p95_ms": 5.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1155}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5850.086, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:24:34+09:00", "last_error_summary": "[h264 @ 0x55a27d044b40] cabac decode of qscale diff failed at 8 41 [h264 @ 0x55a27d044b40] error while decoding MB 8 41, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:24:34+09:00", "last_error_summary": "[h264 @ 0x55a27d044b40] cabac decode of qscale diff failed at 8 41 [h264 @ 0x55a27d044b40] error while decoding MB 8 41, bytestream -9", "last_exit_at": "2026-07-06T17:24:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.061, "last_started_at": "2026-07-06T17:21:25+09:00", "pid": 19660, "process": {"exists": false, "pid": 19660}, "restart_count": 52, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.886, "sample_count": 2123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 138613, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:24:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T17:24:34+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T17:24:33+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T17:24:34.009225+09:00", "mtime_epoch": 1783326274.0092251, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192405}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:24:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1604.547, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.183, "pushed": 7.183, "read": 7.15}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.131, "count": 215, "max_ms": 9.472, "p95_ms": 7.706}, "gstreamer_push": {"avg_ms": 1.116, "count": 215, "max_ms": 1.775, "p95_ms": 1.284}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.113, "count": 215, "max_ms": 86.319, "p95_ms": 76.233}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.705, "count": 215, "max_ms": 120.398, "p95_ms": 105.021}, "yolo11_640_fire_11class": {"avg_ms": 61.511, "count": 215, "max_ms": 88.489, "p95_ms": 80.598}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.197, "count": 215, "max_ms": 97.985, "p95_ms": 91.892}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.224, "count": 215, "max_ms": 180.689, "p95_ms": 162.758}, "read_interval": {"avg_ms": 139.943, "count": 214, "max_ms": 191.749, "p95_ms": 172.521}, "triton_wait": {"avg_ms": 77.059, "count": 215, "max_ms": 120.411, "p95_ms": 105.034}, "visualize": {"avg_ms": 5.111, "count": 215, "max_ms": 6.54, "p95_ms": 5.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1155}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5850.074, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:24:34+09:00", "last_error_summary": "[h264 @ 0x55a27d044b40] cabac decode of qscale diff failed at 8 41 [h264 @ 0x55a27d044b40] error while decoding MB 8 41, bytestream -9", "last_exit_at": "2026-07-06T17:24:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.067, "last_started_at": "2026-07-06T17:21:25+09:00", "pid": 19660, "process": {"exists": false, "pid": 19660}, "restart_count": 52, "restart_count_per_hour": 19, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.886, "sample_count": 2123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 138613, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:24:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T17:24:34+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T17:24:33+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T17:24:34.009225+09:00", "mtime_epoch": 1783326274.0092251, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192405}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:24:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1604.547, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.182, "pushed": 7.182, "read": 7.148}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.131, "count": 215, "max_ms": 9.472, "p95_ms": 7.706}, "gstreamer_push": {"avg_ms": 1.116, "count": 215, "max_ms": 1.775, "p95_ms": 1.284}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.113, "count": 215, "max_ms": 86.319, "p95_ms": 76.233}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.705, "count": 215, "max_ms": 120.398, "p95_ms": 105.021}, "yolo11_640_fire_11class": {"avg_ms": 61.511, "count": 215, "max_ms": 88.489, "p95_ms": 80.598}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.197, "count": 215, "max_ms": 97.985, "p95_ms": 91.892}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.224, "count": 215, "max_ms": 180.689, "p95_ms": 162.758}, "read_interval": {"avg_ms": 139.943, "count": 214, "max_ms": 191.749, "p95_ms": 172.521}, "triton_wait": {"avg_ms": 77.059, "count": 215, "max_ms": 120.411, "p95_ms": 105.034}, "visualize": {"avg_ms": 5.111, "count": 215, "max_ms": 6.54, "p95_ms": 5.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1155}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5949.543, "limit_mb": 8796093022207.996}, "detail": {"pid": 19767, "restart_count": 53, "start_count": 54}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:24:34+09:00", "last_error_summary": "[h264 @ 0x55a27d044b40] cabac decode of qscale diff failed at 8 41 [h264 @ 0x55a27d044b40] error while decoding MB 8 41, bytestream -9", "last_exit_at": "2026-07-06T17:24:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:24:37+09:00", "pid": 19767, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.328, "name": "ffmpeg", "pid": 19767, "rss_mb": 139.328, "state": "S (sleeping)", "threads": 106}, "restart_count": 53, "restart_count_per_hour": 20, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.892, "sample_count": 2088, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 138613, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:24:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.947, "last_frame_pushed_at": "2026-07-06T17:24:34+09:00", "last_frame_read_age_sec": 5.07, "last_frame_read_at": "2026-07-06T17:24:33+09:00", "mjpeg": {"age_sec": 4.927, "exists": true, "mtime": "2026-07-06T17:24:34.141227+09:00", "mtime_epoch": 1783326274.1412268, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192463}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:24:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1604.547, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.011, "pushed": 6.011, "read": 5.978}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.132, "count": 180, "max_ms": 9.472, "p95_ms": 7.809}, "gstreamer_push": {"avg_ms": 1.112, "count": 180, "max_ms": 1.775, "p95_ms": 1.258}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.007, "count": 180, "max_ms": 86.319, "p95_ms": 76.233}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.455, "count": 180, "max_ms": 120.398, "p95_ms": 105.021}, "yolo11_640_fire_11class": {"avg_ms": 61.363, "count": 180, "max_ms": 86.309, "p95_ms": 78.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.928, "count": 180, "max_ms": 97.985, "p95_ms": 91.728}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.94, "count": 180, "max_ms": 180.689, "p95_ms": 162.758}, "read_interval": {"avg_ms": 139.665, "count": 179, "max_ms": 191.749, "p95_ms": 173.09}, "triton_wait": {"avg_ms": 76.781, "count": 180, "max_ms": 120.411, "p95_ms": 105.034}, "visualize": {"avg_ms": 5.108, "count": 180, "max_ms": 6.54, "p95_ms": 5.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5859.91, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563a69d1b440] left block unavailable for requested intra mode [h264 @ 0x563a69d1b440] error while decoding MB 0 37, bytestream 5477 [h264 @ 0x563a693e7700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a693e7700] error while decoding MB 0 42, bytestream 2161 [h264 @ 0x563a693ea2c0] left block unavailable for requested intra mode [h264 @ 0x563a693ea2c0] error while decoding MB 0 35, bytestream 6967 [h264 @ 0x563a69f87fc0] left block unavailable for requested intra m", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 220, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:28:28+09:00", "last_error_summary": "[h264 @ 0x563a69d1b440] left block unavailable for requested intra mode [h264 @ 0x563a69d1b440] error while decoding MB 0 37, bytestream 5477 [h264 @ 0x563a693e7700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a693e7700] error while decoding MB 0 42, bytestream 2161 [h264 @ 0x563a693ea2c0] left block unavailable for requested intra mode [h264 @ 0x563a693ea2c0] error while decoding MB 0 35, bytestream 6967 [h264 @ 0x563a69f87fc0] left block unavailable for requested intra m", "last_exit_at": "2026-07-06T17:24:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 231.492, "last_started_at": "2026-07-06T17:24:37+09:00", "pid": 19767, "process": {"exists": false, "pid": 19767}, "restart_count": 53, "restart_count_per_hour": 20, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.999, "sample_count": 1950, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 140090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:28:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T17:28:28+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T17:28:28+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T17:28:28.512472+09:00", "mtime_epoch": 1783326508.5124722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191144}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:28:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1605.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 73, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.437, "pushed": 2.437, "read": 2.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.707, "count": 72, "max_ms": 8.96, "p95_ms": 7.636}, "gstreamer_push": {"avg_ms": 1.062, "count": 73, "max_ms": 1.723, "p95_ms": 1.203}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.5, "count": 72, "max_ms": 78.876, "p95_ms": 73.473}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.698, "count": 72, "max_ms": 105.639, "p95_ms": 102.555}, "yolo11_640_fire_11class": {"avg_ms": 60.293, "count": 72, "max_ms": 85.883, "p95_ms": 81.742}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.304, "count": 72, "max_ms": 97.301, "p95_ms": 95.936}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.526, "count": 73, "max_ms": 181.102, "p95_ms": 179.177}, "read_interval": {"avg_ms": 412.013, "count": 73, "max_ms": 19208.012, "p95_ms": 187.952}, "triton_wait": {"avg_ms": 75.71, "count": 72, "max_ms": 105.652, "p95_ms": 102.569}, "visualize": {"avg_ms": 5.752, "count": 72, "max_ms": 7.194, "p95_ms": 6.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5858.688, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:28:28+09:00", "last_error_summary": "[h264 @ 0x563a69d1b440] left block unavailable for requested intra mode [h264 @ 0x563a69d1b440] error while decoding MB 0 37, bytestream 5477 [h264 @ 0x563a693e7700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a693e7700] error while decoding MB 0 42, bytestream 2161 [h264 @ 0x563a693ea2c0] left block unavailable for requested intra mode [h264 @ 0x563a693ea2c0] error while decoding MB 0 35, bytestream 6967 [h264 @ 0x563a69f87fc0] left block unavailable for requested intra m", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 220, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:28:28+09:00", "last_error_summary": "[h264 @ 0x563a69d1b440] left block unavailable for requested intra mode [h264 @ 0x563a69d1b440] error while decoding MB 0 37, bytestream 5477 [h264 @ 0x563a693e7700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a693e7700] error while decoding MB 0 42, bytestream 2161 [h264 @ 0x563a693ea2c0] left block unavailable for requested intra mode [h264 @ 0x563a693ea2c0] error while decoding MB 0 35, bytestream 6967 [h264 @ 0x563a69f87fc0] left block unavailable for requested intra m", "last_exit_at": "2026-07-06T17:28:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 231.582, "last_started_at": "2026-07-06T17:24:37+09:00", "pid": 19767, "process": {"exists": false, "pid": 19767}, "restart_count": 53, "restart_count_per_hour": 20, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.953, "sample_count": 1951, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 140091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:28:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T17:28:28+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T17:28:28+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-06T17:28:28.512472+09:00", "mtime_epoch": 1783326508.5124722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191144}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:28:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1605.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.442, "pushed": 2.442, "read": 2.409}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.697, "count": 73, "max_ms": 8.96, "p95_ms": 7.636}, "gstreamer_push": {"avg_ms": 1.062, "count": 73, "max_ms": 1.723, "p95_ms": 1.203}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.69, "count": 73, "max_ms": 78.876, "p95_ms": 73.473}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.697, "count": 73, "max_ms": 105.639, "p95_ms": 102.555}, "yolo11_640_fire_11class": {"avg_ms": 60.398, "count": 73, "max_ms": 85.883, "p95_ms": 81.742}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.475, "count": 73, "max_ms": 97.301, "p95_ms": 95.936}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.984, "count": 73, "max_ms": 181.102, "p95_ms": 179.177}, "read_interval": {"avg_ms": 415.19, "count": 72, "max_ms": 19208.012, "p95_ms": 187.952}, "triton_wait": {"avg_ms": 75.78, "count": 73, "max_ms": 105.652, "p95_ms": 102.569}, "visualize": {"avg_ms": 5.743, "count": 73, "max_ms": 7.194, "p95_ms": 6.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5858.676, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 220, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:28:28+09:00", "last_error_summary": "[h264 @ 0x563a69d1b440] left block unavailable for requested intra mode [h264 @ 0x563a69d1b440] error while decoding MB 0 37, bytestream 5477 [h264 @ 0x563a693e7700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a693e7700] error while decoding MB 0 42, bytestream 2161 [h264 @ 0x563a693ea2c0] left block unavailable for requested intra mode [h264 @ 0x563a693ea2c0] error while decoding MB 0 35, bytestream 6967 [h264 @ 0x563a69f87fc0] left block unavailable for requested intra m", "last_exit_at": "2026-07-06T17:28:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 231.586, "last_started_at": "2026-07-06T17:24:37+09:00", "pid": 19767, "process": {"exists": false, "pid": 19767}, "restart_count": 53, "restart_count_per_hour": 20, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.999, "sample_count": 1950, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 140091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:28:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T17:28:28+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-06T17:28:28+09:00", "mjpeg": {"age_sec": 0.141, "exists": true, "mtime": "2026-07-06T17:28:28.512472+09:00", "mtime_epoch": 1783326508.5124722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191144}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:28:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1605.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.442, "pushed": 2.442, "read": 2.408}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.697, "count": 73, "max_ms": 8.96, "p95_ms": 7.636}, "gstreamer_push": {"avg_ms": 1.062, "count": 73, "max_ms": 1.723, "p95_ms": 1.203}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.69, "count": 73, "max_ms": 78.876, "p95_ms": 73.473}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.697, "count": 73, "max_ms": 105.639, "p95_ms": 102.555}, "yolo11_640_fire_11class": {"avg_ms": 60.398, "count": 73, "max_ms": 85.883, "p95_ms": 81.742}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.475, "count": 73, "max_ms": 97.301, "p95_ms": 95.936}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.984, "count": 73, "max_ms": 181.102, "p95_ms": 179.177}, "read_interval": {"avg_ms": 415.19, "count": 72, "max_ms": 19208.012, "p95_ms": 187.952}, "triton_wait": {"avg_ms": 75.78, "count": 73, "max_ms": 105.652, "p95_ms": 102.569}, "visualize": {"avg_ms": 5.743, "count": 73, "max_ms": 7.194, "p95_ms": 6.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5967.016, "limit_mb": 8796093022207.996}, "detail": {"pid": 19874, "restart_count": 54, "start_count": 55}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 220, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:28:28+09:00", "last_error_summary": "[h264 @ 0x563a69d1b440] left block unavailable for requested intra mode [h264 @ 0x563a69d1b440] error while decoding MB 0 37, bytestream 5477 [h264 @ 0x563a693e7700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a693e7700] error while decoding MB 0 42, bytestream 2161 [h264 @ 0x563a693ea2c0] left block unavailable for requested intra mode [h264 @ 0x563a693ea2c0] error while decoding MB 0 35, bytestream 6967 [h264 @ 0x563a69f87fc0] left block unavailable for requested intra m", "last_exit_at": "2026-07-06T17:28:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:28:30+09:00", "pid": 19874, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.961, "name": "ffmpeg", "pid": 19874, "rss_mb": 138.961, "state": "S (sleeping)", "threads": 106}, "restart_count": 54, "restart_count_per_hour": 21, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.047, "sample_count": 1921, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 140091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:28:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.079, "last_frame_pushed_at": "2026-07-06T17:28:28+09:00", "last_frame_read_age_sec": 4.216, "last_frame_read_at": "2026-07-06T17:28:28+09:00", "mjpeg": {"age_sec": 4.065, "exists": true, "mtime": "2026-07-06T17:28:28.656474+09:00", "mtime_epoch": 1783326508.656474, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191591}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:28:32+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1605.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 48, "pushed_frames": 48, "read_frames": 47, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.602, "pushed": 1.602, "read": 1.569}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.463, "count": 47, "max_ms": 7.457, "p95_ms": 7.402}, "gstreamer_push": {"avg_ms": 1.074, "count": 48, "max_ms": 1.723, "p95_ms": 1.203}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.128, "count": 47, "max_ms": 78.876, "p95_ms": 72.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.845, "count": 47, "max_ms": 105.639, "p95_ms": 102.79}, "yolo11_640_fire_11class": {"avg_ms": 59.777, "count": 47, "max_ms": 85.883, "p95_ms": 79.376}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.794, "count": 47, "max_ms": 97.126, "p95_ms": 95.936}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.795, "count": 48, "max_ms": 179.177, "p95_ms": 168.077}, "read_interval": {"avg_ms": 551.079, "count": 47, "max_ms": 19208.012, "p95_ms": 187.952}, "triton_wait": {"avg_ms": 74.967, "count": 47, "max_ms": 105.652, "p95_ms": 102.801}, "visualize": {"avg_ms": 5.521, "count": 47, "max_ms": 7.194, "p95_ms": 6.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1174}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5853.41, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5611083d14c0] cabac decode of qscale diff failed at 61 44 [h264 @ 0x5611083d14c0] error while decoding MB 61 44, bytestream 3687 [h264 @ 0x56110905ccc0] left block unavailable for requested intra mode [h264 @ 0x56110905ccc0] error while decoding MB 0 42, bytestream 4525 [h264 @ 0x56110832acc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56110832acc0] error while decoding MB 0 22, bytestream 40522 [h264 @ 0x5611083a92c0] error while decoding MB 14 9, bytestream -8 [i", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 225, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:30:57+09:00", "last_error_summary": "[h264 @ 0x5611083d14c0] cabac decode of qscale diff failed at 61 44 [h264 @ 0x5611083d14c0] error while decoding MB 61 44, bytestream 3687 [h264 @ 0x56110905ccc0] left block unavailable for requested intra mode [h264 @ 0x56110905ccc0] error while decoding MB 0 42, bytestream 4525 [h264 @ 0x56110832acc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56110832acc0] error while decoding MB 0 22, bytestream 40522 [h264 @ 0x5611083a92c0] error while decoding MB 14 9, bytestream -8 [i", "last_exit_at": "2026-07-06T17:28:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.321, "last_started_at": "2026-07-06T17:28:30+09:00", "pid": 19874, "process": {"exists": false, "pid": 19874}, "restart_count": 54, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.648, "sample_count": 1809, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 140975, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:30:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T17:30:56+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T17:30:56+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-06T17:30:56.982621+09:00", "mtime_epoch": 1783326656.9826214, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:30:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1605.836, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 80, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.672, "pushed": 2.672, "read": 2.672}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.957, "count": 80, "max_ms": 8.177, "p95_ms": 6.531}, "gstreamer_push": {"avg_ms": 1.113, "count": 80, "max_ms": 1.523, "p95_ms": 1.247}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.013, "count": 80, "max_ms": 75.929, "p95_ms": 71.937}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.541, "count": 80, "max_ms": 104.202, "p95_ms": 99.745}, "yolo11_640_fire_11class": {"avg_ms": 61.004, "count": 80, "max_ms": 79.319, "p95_ms": 76.41}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.119, "count": 80, "max_ms": 94.392, "p95_ms": 86.991}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.834, "count": 80, "max_ms": 156.877, "p95_ms": 152.891}, "read_interval": {"avg_ms": 374.827, "count": 80, "max_ms": 10169.551, "p95_ms": 164.99}, "triton_wait": {"avg_ms": 76.897, "count": 80, "max_ms": 104.214, "p95_ms": 99.758}, "visualize": {"avg_ms": 5.077, "count": 80, "max_ms": 6.435, "p95_ms": 5.368}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1174}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5852.527, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:30:57+09:00", "last_error_summary": "[h264 @ 0x5611083d14c0] cabac decode of qscale diff failed at 61 44 [h264 @ 0x5611083d14c0] error while decoding MB 61 44, bytestream 3687 [h264 @ 0x56110905ccc0] left block unavailable for requested intra mode [h264 @ 0x56110905ccc0] error while decoding MB 0 42, bytestream 4525 [h264 @ 0x56110832acc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56110832acc0] error while decoding MB 0 22, bytestream 40522 [h264 @ 0x5611083a92c0] error while decoding MB 14 9, bytestream -8 [i", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 225, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:30:57+09:00", "last_error_summary": "[h264 @ 0x5611083d14c0] cabac decode of qscale diff failed at 61 44 [h264 @ 0x5611083d14c0] error while decoding MB 61 44, bytestream 3687 [h264 @ 0x56110905ccc0] left block unavailable for requested intra mode [h264 @ 0x56110905ccc0] error while decoding MB 0 42, bytestream 4525 [h264 @ 0x56110832acc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56110832acc0] error while decoding MB 0 22, bytestream 40522 [h264 @ 0x5611083a92c0] error while decoding MB 14 9, bytestream -8 [i", "last_exit_at": "2026-07-06T17:30:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.391, "last_started_at": "2026-07-06T17:28:30+09:00", "pid": 19874, "process": {"exists": false, "pid": 19874}, "restart_count": 54, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.648, "sample_count": 1810, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 140976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:30:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T17:30:57+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T17:30:56+09:00", "mjpeg": {"age_sec": 0.129, "exists": true, "mtime": "2026-07-06T17:30:56.982621+09:00", "mtime_epoch": 1783326656.9826214, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:30:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1605.836, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 80, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.667, "pushed": 2.667, "read": 2.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.957, "count": 80, "max_ms": 8.177, "p95_ms": 6.531}, "gstreamer_push": {"avg_ms": 1.113, "count": 80, "max_ms": 1.523, "p95_ms": 1.247}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.168, "count": 80, "max_ms": 75.929, "p95_ms": 71.937}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.651, "count": 80, "max_ms": 104.202, "p95_ms": 99.745}, "yolo11_640_fire_11class": {"avg_ms": 61.044, "count": 80, "max_ms": 79.319, "p95_ms": 76.41}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.268, "count": 80, "max_ms": 94.392, "p95_ms": 86.991}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.972, "count": 80, "max_ms": 156.877, "p95_ms": 152.891}, "read_interval": {"avg_ms": 374.827, "count": 80, "max_ms": 10169.551, "p95_ms": 164.99}, "triton_wait": {"avg_ms": 77.007, "count": 80, "max_ms": 104.214, "p95_ms": 99.758}, "visualize": {"avg_ms": 5.078, "count": 80, "max_ms": 6.435, "p95_ms": 5.368}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1174}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5852.516, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 225, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:30:57+09:00", "last_error_summary": "[h264 @ 0x5611083d14c0] cabac decode of qscale diff failed at 61 44 [h264 @ 0x5611083d14c0] error while decoding MB 61 44, bytestream 3687 [h264 @ 0x56110905ccc0] left block unavailable for requested intra mode [h264 @ 0x56110905ccc0] error while decoding MB 0 42, bytestream 4525 [h264 @ 0x56110832acc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56110832acc0] error while decoding MB 0 22, bytestream 40522 [h264 @ 0x5611083a92c0] error while decoding MB 14 9, bytestream -8 [i", "last_exit_at": "2026-07-06T17:30:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.398, "last_started_at": "2026-07-06T17:28:30+09:00", "pid": 19874, "process": {"exists": false, "pid": 19874}, "restart_count": 54, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.648, "sample_count": 1810, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 140976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:30:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T17:30:57+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-06T17:30:56+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-06T17:30:56.982621+09:00", "mtime_epoch": 1783326656.9826214, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:30:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1605.836, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 79, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.677, "pushed": 2.677, "read": 2.643}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.957, "count": 80, "max_ms": 8.177, "p95_ms": 6.531}, "gstreamer_push": {"avg_ms": 1.113, "count": 80, "max_ms": 1.523, "p95_ms": 1.247}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.168, "count": 80, "max_ms": 75.929, "p95_ms": 71.937}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.651, "count": 80, "max_ms": 104.202, "p95_ms": 99.745}, "yolo11_640_fire_11class": {"avg_ms": 61.044, "count": 80, "max_ms": 79.319, "p95_ms": 76.41}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.268, "count": 80, "max_ms": 94.392, "p95_ms": 86.991}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.972, "count": 80, "max_ms": 156.877, "p95_ms": 152.891}, "read_interval": {"avg_ms": 378.007, "count": 79, "max_ms": 10169.551, "p95_ms": 166.348}, "triton_wait": {"avg_ms": 77.007, "count": 80, "max_ms": 104.214, "p95_ms": 99.758}, "visualize": {"avg_ms": 5.078, "count": 80, "max_ms": 6.435, "p95_ms": 5.368}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1174}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5869.723, "limit_mb": 8796093022207.996}, "detail": {"pid": 19981, "restart_count": 55, "start_count": 56}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 225, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:30:57+09:00", "last_error_summary": "[h264 @ 0x5611083d14c0] cabac decode of qscale diff failed at 61 44 [h264 @ 0x5611083d14c0] error while decoding MB 61 44, bytestream 3687 [h264 @ 0x56110905ccc0] left block unavailable for requested intra mode [h264 @ 0x56110905ccc0] error while decoding MB 0 42, bytestream 4525 [h264 @ 0x56110832acc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56110832acc0] error while decoding MB 0 22, bytestream 40522 [h264 @ 0x5611083a92c0] error while decoding MB 14 9, bytestream -8 [i", "last_exit_at": "2026-07-06T17:31:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:31:00+09:00", "pid": 19981, "process": {"exists": true, "fd_count": 4, "hwm_mb": 52.926, "name": "ffmpeg", "pid": 19981, "rss_mb": 52.926, "state": "S (sleeping)", "threads": 1}, "restart_count": 55, "restart_count_per_hour": 21, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.776, "sample_count": 1775, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 140976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:30:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.992, "last_frame_pushed_at": "2026-07-06T17:30:57+09:00", "last_frame_read_age_sec": 5.118, "last_frame_read_at": "2026-07-06T17:30:56+09:00", "mjpeg": {"age_sec": 4.979, "exists": true, "mtime": "2026-07-06T17:30:57.118623+09:00", "mtime_epoch": 1783326657.1186235, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192639}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:31:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1605.836, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.505, "pushed": 1.505, "read": 1.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.994, "count": 45, "max_ms": 6.599, "p95_ms": 6.271}, "gstreamer_push": {"avg_ms": 1.104, "count": 45, "max_ms": 1.319, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.384, "count": 45, "max_ms": 75.929, "p95_ms": 74.853}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.093, "count": 45, "max_ms": 104.202, "p95_ms": 97.642}, "yolo11_640_fire_11class": {"avg_ms": 61.098, "count": 45, "max_ms": 78.495, "p95_ms": 75.942}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.749, "count": 45, "max_ms": 93.881, "p95_ms": 86.787}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.306, "count": 45, "max_ms": 156.877, "p95_ms": 150.558}, "read_interval": {"avg_ms": 565.837, "count": 44, "max_ms": 10169.551, "p95_ms": 166.579}, "triton_wait": {"avg_ms": 74.403, "count": 45, "max_ms": 104.214, "p95_ms": 97.658}, "visualize": {"avg_ms": 5.085, "count": 45, "max_ms": 6.037, "p95_ms": 5.368}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1185}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5859.039, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 226, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:34:07+09:00", "last_error_summary": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "last_exit_at": "2026-07-06T17:31:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.434, "last_started_at": "2026-07-06T17:31:00+09:00", "pid": 19981, "process": {"exists": false, "pid": 19981}, "restart_count": 55, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.218, "sample_count": 1952, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 142290, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:34:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T17:34:07+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T17:34:07+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-06T17:34:07.485452+09:00", "mtime_epoch": 1783326847.4854522, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:34:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1606.094, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.887, "pushed": 6.887, "read": 6.887}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.483, "count": 206, "max_ms": 9.708, "p95_ms": 8.162}, "gstreamer_push": {"avg_ms": 1.09, "count": 206, "max_ms": 1.385, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.711, "count": 206, "max_ms": 84.562, "p95_ms": 75.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.236, "count": 206, "max_ms": 119.511, "p95_ms": 99.877}, "yolo11_640_fire_11class": {"avg_ms": 60.63, "count": 206, "max_ms": 86.888, "p95_ms": 77.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.979, "count": 206, "max_ms": 100.025, "p95_ms": 91.916}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.38, "count": 206, "max_ms": 199.977, "p95_ms": 178.979}, "read_interval": {"avg_ms": 145.695, "count": 206, "max_ms": 207.989, "p95_ms": 187.31}, "triton_wait": {"avg_ms": 76.37, "count": 206, "max_ms": 119.523, "p95_ms": 99.896}, "visualize": {"avg_ms": 5.478, "count": 206, "max_ms": 8.325, "p95_ms": 7.319}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1185}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5858.555, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:34:07+09:00", "last_error_summary": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 226, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:34:07+09:00", "last_error_summary": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "last_exit_at": "2026-07-06T17:34:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.505, "last_started_at": "2026-07-06T17:31:00+09:00", "pid": 19981, "process": {"exists": false, "pid": 19981}, "restart_count": 55, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.218, "sample_count": 1952, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 142291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:34:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T17:34:07+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-06T17:34:07+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T17:34:07.485452+09:00", "mtime_epoch": 1783326847.4854522, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:34:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1606.094, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.904, "pushed": 6.904, "read": 6.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.477, "count": 206, "max_ms": 9.708, "p95_ms": 7.909}, "gstreamer_push": {"avg_ms": 1.09, "count": 207, "max_ms": 1.385, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.764, "count": 206, "max_ms": 84.562, "p95_ms": 75.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.292, "count": 206, "max_ms": 119.511, "p95_ms": 99.877}, "yolo11_640_fire_11class": {"avg_ms": 60.727, "count": 206, "max_ms": 86.888, "p95_ms": 77.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.021, "count": 206, "max_ms": 100.025, "p95_ms": 91.916}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.29, "count": 207, "max_ms": 199.977, "p95_ms": 178.979}, "read_interval": {"avg_ms": 145.695, "count": 206, "max_ms": 207.989, "p95_ms": 187.31}, "triton_wait": {"avg_ms": 76.426, "count": 206, "max_ms": 119.523, "p95_ms": 99.896}, "visualize": {"avg_ms": 5.466, "count": 206, "max_ms": 8.325, "p95_ms": 7.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1185}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5858.547, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 226, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:34:07+09:00", "last_error_summary": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "last_exit_at": "2026-07-06T17:34:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.511, "last_started_at": "2026-07-06T17:31:00+09:00", "pid": 19981, "process": {"exists": false, "pid": 19981}, "restart_count": 55, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.218, "sample_count": 1952, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 142291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:34:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T17:34:07+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T17:34:07+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T17:34:07.485452+09:00", "mtime_epoch": 1783326847.4854522, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:34:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1606.094, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.903, "pushed": 6.903, "read": 6.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.477, "count": 206, "max_ms": 9.708, "p95_ms": 7.909}, "gstreamer_push": {"avg_ms": 1.09, "count": 207, "max_ms": 1.385, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.764, "count": 206, "max_ms": 84.562, "p95_ms": 75.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.292, "count": 206, "max_ms": 119.511, "p95_ms": 99.877}, "yolo11_640_fire_11class": {"avg_ms": 60.727, "count": 206, "max_ms": 86.888, "p95_ms": 77.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.021, "count": 206, "max_ms": 100.025, "p95_ms": 91.916}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.29, "count": 207, "max_ms": 199.977, "p95_ms": 178.979}, "read_interval": {"avg_ms": 145.695, "count": 206, "max_ms": 207.989, "p95_ms": 187.31}, "triton_wait": {"avg_ms": 76.426, "count": 206, "max_ms": 119.523, "p95_ms": 99.896}, "visualize": {"avg_ms": 5.466, "count": 206, "max_ms": 8.325, "p95_ms": 7.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1185}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5963.723, "limit_mb": 8796093022207.996}, "detail": {"pid": 20088, "restart_count": 56, "start_count": 57}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 226, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:34:07+09:00", "last_error_summary": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "last_exit_at": "2026-07-06T17:34:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:34:10+09:00", "pid": 20088, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.73, "name": "ffmpeg", "pid": 20088, "rss_mb": 138.73, "state": "S (sleeping)", "threads": 106}, "restart_count": 56, "restart_count_per_hour": 21, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.137, "sample_count": 1919, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 142291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:34:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.871, "last_frame_pushed_at": "2026-07-06T17:34:07+09:00", "last_frame_read_age_sec": 4.991, "last_frame_read_at": "2026-07-06T17:34:07+09:00", "mjpeg": {"age_sec": 4.855, "exists": true, "mtime": "2026-07-06T17:34:07.613454+09:00", "mtime_epoch": 1783326847.613454, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186709}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:34:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1606.094, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.969, "pushed": 5.969, "read": 5.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.296, "count": 178, "max_ms": 9.332, "p95_ms": 7.42}, "gstreamer_push": {"avg_ms": 1.108, "count": 179, "max_ms": 1.385, "p95_ms": 1.223}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.728, "count": 178, "max_ms": 84.562, "p95_ms": 75.254}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.255, "count": 178, "max_ms": 119.511, "p95_ms": 100.556}, "yolo11_640_fire_11class": {"avg_ms": 60.744, "count": 178, "max_ms": 86.888, "p95_ms": 77.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.814, "count": 178, "max_ms": 100.025, "p95_ms": 91.296}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.639, "count": 179, "max_ms": 199.977, "p95_ms": 157.527}, "read_interval": {"avg_ms": 141.221, "count": 178, "max_ms": 207.989, "p95_ms": 168.351}, "triton_wait": {"avg_ms": 76.333, "count": 178, "max_ms": 119.523, "p95_ms": 100.565}, "visualize": {"avg_ms": 5.253, "count": 178, "max_ms": 6.501, "p95_ms": 6.417}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5862.77, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:34:07+09:00", "last_error_summary": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 226, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:34:07+09:00", "last_error_summary": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "last_exit_at": "2026-07-06T17:37:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.48, "last_started_at": "2026-07-06T17:34:10+09:00", "pid": 20088, "process": {"exists": false, "pid": 20088}, "restart_count": 56, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.658, "sample_count": 2100, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143601, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:37:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T17:37:15+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-06T17:37:15+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-06T17:37:15.800308+09:00", "mtime_epoch": 1783327035.8003078, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182064}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:37:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1606.867, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.152, "pushed": 7.152, "read": 7.118}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.278, "count": 214, "max_ms": 9.227, "p95_ms": 7.529}, "gstreamer_push": {"avg_ms": 1.109, "count": 214, "max_ms": 1.4, "p95_ms": 1.221}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.638, "count": 214, "max_ms": 83.442, "p95_ms": 73.401}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.081, "count": 214, "max_ms": 113.728, "p95_ms": 98.789}, "yolo11_640_fire_11class": {"avg_ms": 59.181, "count": 214, "max_ms": 82.33, "p95_ms": 77.062}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.781, "count": 214, "max_ms": 100.485, "p95_ms": 91.6}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.837, "count": 214, "max_ms": 176.463, "p95_ms": 157.954}, "read_interval": {"avg_ms": 140.321, "count": 213, "max_ms": 184.808, "p95_ms": 167.213}, "triton_wait": {"avg_ms": 75.184, "count": 214, "max_ms": 113.738, "p95_ms": 98.8}, "visualize": {"avg_ms": 5.299, "count": 214, "max_ms": 7.392, "p95_ms": 6.33}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5862.762, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 226, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:34:07+09:00", "last_error_summary": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "last_exit_at": "2026-07-06T17:37:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.485, "last_started_at": "2026-07-06T17:34:10+09:00", "pid": 20088, "process": {"exists": false, "pid": 20088}, "restart_count": 56, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.658, "sample_count": 2100, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143601, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:37:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T17:37:15+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-06T17:37:15+09:00", "mjpeg": {"age_sec": 0.152, "exists": true, "mtime": "2026-07-06T17:37:15.800308+09:00", "mtime_epoch": 1783327035.8003078, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182064}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:37:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1606.867, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.15, "pushed": 7.15, "read": 7.117}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.278, "count": 214, "max_ms": 9.227, "p95_ms": 7.529}, "gstreamer_push": {"avg_ms": 1.109, "count": 214, "max_ms": 1.4, "p95_ms": 1.221}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.638, "count": 214, "max_ms": 83.442, "p95_ms": 73.401}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.081, "count": 214, "max_ms": 113.728, "p95_ms": 98.789}, "yolo11_640_fire_11class": {"avg_ms": 59.181, "count": 214, "max_ms": 82.33, "p95_ms": 77.062}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.781, "count": 214, "max_ms": 100.485, "p95_ms": 91.6}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.837, "count": 214, "max_ms": 176.463, "p95_ms": 157.954}, "read_interval": {"avg_ms": 140.321, "count": 213, "max_ms": 184.808, "p95_ms": 167.213}, "triton_wait": {"avg_ms": 75.184, "count": 214, "max_ms": 113.738, "p95_ms": 98.8}, "visualize": {"avg_ms": 5.299, "count": 214, "max_ms": 7.392, "p95_ms": 6.33}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5968.508, "limit_mb": 8796093022207.996}, "detail": {"pid": 20195, "restart_count": 57, "start_count": 58}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 226, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:34:07+09:00", "last_error_summary": "[h264 @ 0x5558e92c2700] error while decoding MB 49 22, bytestream -6", "last_exit_at": "2026-07-06T17:37:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:37:18+09:00", "pid": 20195, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.32, "name": "ffmpeg", "pid": 20195, "rss_mb": 136.32, "state": "S (sleeping)", "threads": 106}, "restart_count": 57, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.826, "sample_count": 2068, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143601, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:37:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.419, "last_frame_pushed_at": "2026-07-06T17:37:15+09:00", "last_frame_read_age_sec": 4.569, "last_frame_read_at": "2026-07-06T17:37:15+09:00", "mjpeg": {"age_sec": 4.405, "exists": true, "mtime": "2026-07-06T17:37:15.956310+09:00", "mtime_epoch": 1783327035.95631, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178800}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:37:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1606.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 182, "pushed_frames": 182, "read_frames": 181, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.092, "pushed": 6.092, "read": 6.058}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.293, "count": 182, "max_ms": 9.227, "p95_ms": 7.427}, "gstreamer_push": {"avg_ms": 1.104, "count": 182, "max_ms": 1.4, "p95_ms": 1.216}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.007, "count": 182, "max_ms": 83.442, "p95_ms": 73.401}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.907, "count": 182, "max_ms": 113.728, "p95_ms": 97.141}, "yolo11_640_fire_11class": {"avg_ms": 59.572, "count": 182, "max_ms": 82.33, "p95_ms": 77.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.07, "count": 182, "max_ms": 100.485, "p95_ms": 91.835}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.055, "count": 182, "max_ms": 176.463, "p95_ms": 157.954}, "read_interval": {"avg_ms": 140.519, "count": 181, "max_ms": 184.808, "p95_ms": 167.213}, "triton_wait": {"avg_ms": 75.021, "count": 182, "max_ms": 113.738, "p95_ms": 97.394}, "visualize": {"avg_ms": 5.32, "count": 182, "max_ms": 7.392, "p95_ms": 6.335}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1206}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5867.742, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562f9851b380] error while decoding MB 14 41, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 227, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:40:15+09:00", "last_error_summary": "[h264 @ 0x562f9851b380] error while decoding MB 14 41, bytestream -11", "last_exit_at": "2026-07-06T17:37:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 176.946, "last_started_at": "2026-07-06T17:37:18+09:00", "pid": 20195, "process": {"exists": false, "pid": 20195}, "restart_count": 57, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.008, "sample_count": 2079, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 144819, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:40:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T17:40:15+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T17:40:15+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T17:40:15.263066+09:00", "mtime_epoch": 1783327215.2630658, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180354}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:40:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1607.383, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 210, "pushed_frames": 210, "read_frames": 210, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.01, "pushed": 7.01, "read": 7.01}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.364, "count": 210, "max_ms": 10.567, "p95_ms": 8.059}, "gstreamer_push": {"avg_ms": 1.109, "count": 210, "max_ms": 1.476, "p95_ms": 1.267}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.991, "count": 210, "max_ms": 85.453, "p95_ms": 75.943}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.994, "count": 210, "max_ms": 118.775, "p95_ms": 103.828}, "yolo11_640_fire_11class": {"avg_ms": 59.882, "count": 210, "max_ms": 88.596, "p95_ms": 77.37}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.086, "count": 210, "max_ms": 101.716, "p95_ms": 91.903}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.192, "count": 210, "max_ms": 216.813, "p95_ms": 162.416}, "read_interval": {"avg_ms": 142.914, "count": 210, "max_ms": 231.611, "p95_ms": 172.131}, "triton_wait": {"avg_ms": 77.043, "count": 210, "max_ms": 118.787, "p95_ms": 103.846}, "visualize": {"avg_ms": 5.275, "count": 210, "max_ms": 8.789, "p95_ms": 6.313}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1206}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5855.684, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:40:15+09:00", "last_error_summary": "[h264 @ 0x562f9851b380] error while decoding MB 14 41, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 227, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:40:15+09:00", "last_error_summary": "[h264 @ 0x562f9851b380] error while decoding MB 14 41, bytestream -11", "last_exit_at": "2026-07-06T17:40:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 177.043, "last_started_at": "2026-07-06T17:37:18+09:00", "pid": 20195, "process": {"exists": false, "pid": 20195}, "restart_count": 57, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.008, "sample_count": 2080, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 144820, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:40:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T17:40:15+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-06T17:40:15+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-06T17:40:15.263066+09:00", "mtime_epoch": 1783327215.2630658, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180354}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:40:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1607.383, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 210, "pushed_frames": 210, "read_frames": 209, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.025, "pushed": 7.025, "read": 6.992}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.371, "count": 210, "max_ms": 10.567, "p95_ms": 8.059}, "gstreamer_push": {"avg_ms": 1.11, "count": 210, "max_ms": 1.476, "p95_ms": 1.267}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.994, "count": 210, "max_ms": 85.453, "p95_ms": 75.943}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.098, "count": 210, "max_ms": 118.775, "p95_ms": 103.828}, "yolo11_640_fire_11class": {"avg_ms": 59.955, "count": 210, "max_ms": 88.596, "p95_ms": 77.37}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.192, "count": 210, "max_ms": 101.716, "p95_ms": 91.903}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.368, "count": 210, "max_ms": 216.813, "p95_ms": 162.416}, "read_interval": {"avg_ms": 143.06, "count": 209, "max_ms": 231.611, "p95_ms": 172.131}, "triton_wait": {"avg_ms": 77.146, "count": 210, "max_ms": 118.787, "p95_ms": 103.846}, "visualize": {"avg_ms": 5.283, "count": 210, "max_ms": 8.789, "p95_ms": 6.313}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1206}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5855.754, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 227, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:40:15+09:00", "last_error_summary": "[h264 @ 0x562f9851b380] error while decoding MB 14 41, bytestream -11", "last_exit_at": "2026-07-06T17:40:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 177.048, "last_started_at": "2026-07-06T17:37:18+09:00", "pid": 20195, "process": {"exists": false, "pid": 20195}, "restart_count": 57, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.008, "sample_count": 2080, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 144820, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:40:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T17:40:15+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-06T17:40:15+09:00", "mjpeg": {"age_sec": 0.145, "exists": true, "mtime": "2026-07-06T17:40:15.263066+09:00", "mtime_epoch": 1783327215.2630658, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180354}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:40:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1607.383, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 210, "pushed_frames": 210, "read_frames": 209, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.024, "pushed": 7.024, "read": 6.99}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.371, "count": 210, "max_ms": 10.567, "p95_ms": 8.059}, "gstreamer_push": {"avg_ms": 1.11, "count": 210, "max_ms": 1.476, "p95_ms": 1.267}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.994, "count": 210, "max_ms": 85.453, "p95_ms": 75.943}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.098, "count": 210, "max_ms": 118.775, "p95_ms": 103.828}, "yolo11_640_fire_11class": {"avg_ms": 59.955, "count": 210, "max_ms": 88.596, "p95_ms": 77.37}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.192, "count": 210, "max_ms": 101.716, "p95_ms": 91.903}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.368, "count": 210, "max_ms": 216.813, "p95_ms": 162.416}, "read_interval": {"avg_ms": 143.06, "count": 209, "max_ms": 231.611, "p95_ms": 172.131}, "triton_wait": {"avg_ms": 77.146, "count": 210, "max_ms": 118.787, "p95_ms": 103.846}, "visualize": {"avg_ms": 5.283, "count": 210, "max_ms": 8.789, "p95_ms": 6.313}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1206}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5980.441, "limit_mb": 8796093022207.996}, "detail": {"pid": 20302, "restart_count": 58, "start_count": 59}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 227, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:40:15+09:00", "last_error_summary": "[h264 @ 0x562f9851b380] error while decoding MB 14 41, bytestream -11", "last_exit_at": "2026-07-06T17:40:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:40:17+09:00", "pid": 20302, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.297, "name": "ffmpeg", "pid": 20302, "rss_mb": 144.297, "state": "S (sleeping)", "threads": 106}, "restart_count": 58, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.275, "sample_count": 2049, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 144820, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:40:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.219, "last_frame_pushed_at": "2026-07-06T17:40:15+09:00", "last_frame_read_age_sec": 4.363, "last_frame_read_at": "2026-07-06T17:40:15+09:00", "mjpeg": {"age_sec": 4.202, "exists": true, "mtime": "2026-07-06T17:40:15.415068+09:00", "mtime_epoch": 1783327215.4150681, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:40:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1607.383, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.995, "pushed": 5.995, "read": 5.961}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.444, "count": 179, "max_ms": 10.567, "p95_ms": 8.037}, "gstreamer_push": {"avg_ms": 1.119, "count": 179, "max_ms": 1.476, "p95_ms": 1.291}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.798, "count": 179, "max_ms": 85.453, "p95_ms": 75.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.39, "count": 179, "max_ms": 118.775, "p95_ms": 104.978}, "yolo11_640_fire_11class": {"avg_ms": 60.154, "count": 179, "max_ms": 88.596, "p95_ms": 77.607}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.396, "count": 179, "max_ms": 101.716, "p95_ms": 91.903}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.408, "count": 179, "max_ms": 216.813, "p95_ms": 164.635}, "read_interval": {"avg_ms": 144.083, "count": 178, "max_ms": 231.611, "p95_ms": 175.274}, "triton_wait": {"avg_ms": 77.44, "count": 179, "max_ms": 118.787, "p95_ms": 104.993}, "visualize": {"avg_ms": 5.343, "count": 179, "max_ms": 8.789, "p95_ms": 6.384}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.858, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5864.594, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56185cd87300] left block unavailable for requested intra mode [h264 @ 0x56185cd87300] error while decoding MB 0 16, bytestream 120006 [h264 @ 0x56185d9427c0] error while decoding MB 44 16, bytestream -30", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 229, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:43:24+09:00", "last_error_summary": "[h264 @ 0x56185cd87300] left block unavailable for requested intra mode [h264 @ 0x56185cd87300] error while decoding MB 0 16, bytestream 120006 [h264 @ 0x56185d9427c0] error while decoding MB 44 16, bytestream -30", "last_exit_at": "2026-07-06T17:40:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.114, "last_started_at": "2026-07-06T17:40:17+09:00", "pid": 20302, "process": {"exists": false, "pid": 20302}, "restart_count": 58, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.35, "sample_count": 2111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 146148, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:43:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T17:43:24+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T17:43:24+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T17:43:24.690004+09:00", "mtime_epoch": 1783327404.6900043, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183576}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:43:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1608.156, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.242, "pushed": 7.242, "read": 7.242}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.167, "count": 216, "max_ms": 9.1, "p95_ms": 7.424}, "gstreamer_push": {"avg_ms": 1.121, "count": 217, "max_ms": 1.71, "p95_ms": 1.238}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.383, "count": 216, "max_ms": 88.123, "p95_ms": 73.81}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.473, "count": 216, "max_ms": 120.431, "p95_ms": 101.347}, "yolo11_640_fire_11class": {"avg_ms": 60.518, "count": 216, "max_ms": 88.108, "p95_ms": 79.343}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.788, "count": 216, "max_ms": 100.214, "p95_ms": 94.031}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.641, "count": 217, "max_ms": 172.864, "p95_ms": 155.562}, "read_interval": {"avg_ms": 138.375, "count": 217, "max_ms": 182.705, "p95_ms": 165.925}, "triton_wait": {"avg_ms": 75.604, "count": 216, "max_ms": 120.444, "p95_ms": 101.359}, "visualize": {"avg_ms": 5.19, "count": 216, "max_ms": 7.748, "p95_ms": 5.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5864.07, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:43:24+09:00", "last_error_summary": "[h264 @ 0x56185cd87300] left block unavailable for requested intra mode [h264 @ 0x56185cd87300] error while decoding MB 0 16, bytestream 120006 [h264 @ 0x56185d9427c0] error while decoding MB 44 16, bytestream -30", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 229, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:43:24+09:00", "last_error_summary": "[h264 @ 0x56185cd87300] left block unavailable for requested intra mode [h264 @ 0x56185cd87300] error while decoding MB 0 16, bytestream 120006 [h264 @ 0x56185d9427c0] error while decoding MB 44 16, bytestream -30", "last_exit_at": "2026-07-06T17:43:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.194, "last_started_at": "2026-07-06T17:40:17+09:00", "pid": 20302, "process": {"exists": false, "pid": 20302}, "restart_count": 58, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.35, "sample_count": 2111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 146149, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:43:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T17:43:24+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T17:43:24+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T17:43:24.690004+09:00", "mtime_epoch": 1783327404.6900043, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183576}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:43:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1608.156, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.264, "pushed": 7.264, "read": 7.231}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.166, "count": 217, "max_ms": 9.1, "p95_ms": 7.424}, "gstreamer_push": {"avg_ms": 1.122, "count": 217, "max_ms": 1.71, "p95_ms": 1.251}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.343, "count": 217, "max_ms": 88.123, "p95_ms": 73.81}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.468, "count": 217, "max_ms": 120.431, "p95_ms": 101.347}, "yolo11_640_fire_11class": {"avg_ms": 60.455, "count": 217, "max_ms": 88.108, "p95_ms": 79.343}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.74, "count": 217, "max_ms": 100.214, "p95_ms": 94.031}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.68, "count": 217, "max_ms": 172.864, "p95_ms": 155.562}, "read_interval": {"avg_ms": 138.427, "count": 216, "max_ms": 182.705, "p95_ms": 165.925}, "triton_wait": {"avg_ms": 75.599, "count": 217, "max_ms": 120.444, "p95_ms": 101.359}, "visualize": {"avg_ms": 5.19, "count": 217, "max_ms": 7.748, "p95_ms": 5.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5864.062, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 229, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:43:24+09:00", "last_error_summary": "[h264 @ 0x56185cd87300] left block unavailable for requested intra mode [h264 @ 0x56185cd87300] error while decoding MB 0 16, bytestream 120006 [h264 @ 0x56185d9427c0] error while decoding MB 44 16, bytestream -30", "last_exit_at": "2026-07-06T17:43:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.2, "last_started_at": "2026-07-06T17:40:17+09:00", "pid": 20302, "process": {"exists": false, "pid": 20302}, "restart_count": 58, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.35, "sample_count": 2111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 146149, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:43:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T17:43:24+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T17:43:24+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-06T17:43:24.690004+09:00", "mtime_epoch": 1783327404.6900043, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183576}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:43:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1608.156, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.263, "pushed": 7.263, "read": 7.229}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.166, "count": 217, "max_ms": 9.1, "p95_ms": 7.424}, "gstreamer_push": {"avg_ms": 1.122, "count": 217, "max_ms": 1.71, "p95_ms": 1.251}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.343, "count": 217, "max_ms": 88.123, "p95_ms": 73.81}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.468, "count": 217, "max_ms": 120.431, "p95_ms": 101.347}, "yolo11_640_fire_11class": {"avg_ms": 60.455, "count": 217, "max_ms": 88.108, "p95_ms": 79.343}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.74, "count": 217, "max_ms": 100.214, "p95_ms": 94.031}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.68, "count": 217, "max_ms": 172.864, "p95_ms": 155.562}, "read_interval": {"avg_ms": 138.427, "count": 216, "max_ms": 182.705, "p95_ms": 165.925}, "triton_wait": {"avg_ms": 75.599, "count": 217, "max_ms": 120.444, "p95_ms": 101.359}, "visualize": {"avg_ms": 5.19, "count": 217, "max_ms": 7.748, "p95_ms": 5.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5880.254, "limit_mb": 8796093022207.996}, "detail": {"pid": 20409, "restart_count": 59, "start_count": 60}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 229, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:43:24+09:00", "last_error_summary": "[h264 @ 0x56185cd87300] left block unavailable for requested intra mode [h264 @ 0x56185cd87300] error while decoding MB 0 16, bytestream 120006 [h264 @ 0x56185d9427c0] error while decoding MB 44 16, bytestream -30", "last_exit_at": "2026-07-06T17:43:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:43:27+09:00", "pid": 20409, "process": {"exists": true, "fd_count": 4, "hwm_mb": 52.816, "name": "ffmpeg", "pid": 20409, "rss_mb": 52.816, "state": "S (sleeping)", "threads": 1}, "restart_count": 59, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.518, "sample_count": 2077, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 146149, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:43:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.789, "last_frame_pushed_at": "2026-07-06T17:43:24+09:00", "last_frame_read_age_sec": 4.917, "last_frame_read_at": "2026-07-06T17:43:24+09:00", "mjpeg": {"age_sec": 4.773, "exists": true, "mtime": "2026-07-06T17:43:24.822006+09:00", "mtime_epoch": 1783327404.8220065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183551}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:43:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1608.156, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.1, "pushed": 6.1, "read": 6.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.152, "count": 182, "max_ms": 9.1, "p95_ms": 7.28}, "gstreamer_push": {"avg_ms": 1.124, "count": 183, "max_ms": 1.71, "p95_ms": 1.251}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.733, "count": 182, "max_ms": 88.123, "p95_ms": 73.81}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.667, "count": 182, "max_ms": 120.431, "p95_ms": 100.693}, "yolo11_640_fire_11class": {"avg_ms": 60.291, "count": 182, "max_ms": 88.108, "p95_ms": 78.056}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.547, "count": 182, "max_ms": 100.214, "p95_ms": 92.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.687, "count": 183, "max_ms": 172.864, "p95_ms": 153.548}, "read_interval": {"avg_ms": 138.381, "count": 182, "max_ms": 182.705, "p95_ms": 164.679}, "triton_wait": {"avg_ms": 75.762, "count": 182, "max_ms": 120.444, "p95_ms": 100.706}, "visualize": {"avg_ms": 5.18, "count": 182, "max_ms": 7.748, "p95_ms": 5.838}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1229}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5872.844, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557dbb898a80] error while decoding MB 15 40, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 230, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:46:38+09:00", "last_error_summary": "[h264 @ 0x557dbb898a80] error while decoding MB 15 40, bytestream -9", "last_exit_at": "2026-07-06T17:43:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.425, "last_started_at": "2026-07-06T17:43:27+09:00", "pid": 20409, "process": {"exists": false, "pid": 20409}, "restart_count": 59, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.242, "sample_count": 2125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 147502, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:46:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-06T17:46:37+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T17:46:37+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-06T17:46:37.969023+09:00", "mtime_epoch": 1783327597.9690225, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183949}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:46:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1608.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.123, "pushed": 7.123, "read": 7.123}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.329, "count": 213, "max_ms": 9.762, "p95_ms": 8.776}, "gstreamer_push": {"avg_ms": 1.121, "count": 213, "max_ms": 1.954, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.849, "count": 213, "max_ms": 82.221, "p95_ms": 76.093}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.094, "count": 213, "max_ms": 117.276, "p95_ms": 99.474}, "yolo11_640_fire_11class": {"avg_ms": 59.295, "count": 213, "max_ms": 83.996, "p95_ms": 78.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.894, "count": 213, "max_ms": 97.967, "p95_ms": 92.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.923, "count": 213, "max_ms": 187.859, "p95_ms": 160.343}, "read_interval": {"avg_ms": 140.705, "count": 213, "max_ms": 196.256, "p95_ms": 170.96}, "triton_wait": {"avg_ms": 76.24, "count": 213, "max_ms": 117.29, "p95_ms": 99.486}, "visualize": {"avg_ms": 5.366, "count": 213, "max_ms": 9.007, "p95_ms": 6.494}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1229}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5872.102, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:46:38+09:00", "last_error_summary": "[h264 @ 0x557dbb898a80] error while decoding MB 15 40, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 230, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:46:38+09:00", "last_error_summary": "[h264 @ 0x557dbb898a80] error while decoding MB 15 40, bytestream -9", "last_exit_at": "2026-07-06T17:46:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.495, "last_started_at": "2026-07-06T17:43:27+09:00", "pid": 20409, "process": {"exists": false, "pid": 20409}, "restart_count": 59, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.242, "sample_count": 2126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 147503, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:46:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T17:46:38+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-06T17:46:37+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T17:46:37.969023+09:00", "mtime_epoch": 1783327597.9690225, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183949}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:46:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1608.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.139, "pushed": 7.139, "read": 7.106}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.329, "count": 213, "max_ms": 9.762, "p95_ms": 8.776}, "gstreamer_push": {"avg_ms": 1.121, "count": 214, "max_ms": 1.954, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.783, "count": 213, "max_ms": 82.221, "p95_ms": 76.093}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.07, "count": 213, "max_ms": 117.276, "p95_ms": 99.474}, "yolo11_640_fire_11class": {"avg_ms": 59.256, "count": 213, "max_ms": 83.996, "p95_ms": 78.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.869, "count": 213, "max_ms": 97.967, "p95_ms": 92.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.871, "count": 214, "max_ms": 187.859, "p95_ms": 160.343}, "read_interval": {"avg_ms": 140.705, "count": 213, "max_ms": 196.256, "p95_ms": 170.96}, "triton_wait": {"avg_ms": 76.216, "count": 213, "max_ms": 117.29, "p95_ms": 99.486}, "visualize": {"avg_ms": 5.377, "count": 213, "max_ms": 9.007, "p95_ms": 6.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1229}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5872.094, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 230, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:46:38+09:00", "last_error_summary": "[h264 @ 0x557dbb898a80] error while decoding MB 15 40, bytestream -9", "last_exit_at": "2026-07-06T17:46:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.501, "last_started_at": "2026-07-06T17:43:27+09:00", "pid": 20409, "process": {"exists": false, "pid": 20409}, "restart_count": 59, "restart_count_per_hour": 19, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.242, "sample_count": 2126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 147503, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:46:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T17:46:38+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-06T17:46:37+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-06T17:46:37.969023+09:00", "mtime_epoch": 1783327597.9690225, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183949}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:46:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1608.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.138, "pushed": 7.138, "read": 7.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.329, "count": 213, "max_ms": 9.762, "p95_ms": 8.776}, "gstreamer_push": {"avg_ms": 1.121, "count": 214, "max_ms": 1.954, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.783, "count": 213, "max_ms": 82.221, "p95_ms": 76.093}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.07, "count": 213, "max_ms": 117.276, "p95_ms": 99.474}, "yolo11_640_fire_11class": {"avg_ms": 59.256, "count": 213, "max_ms": 83.996, "p95_ms": 78.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.869, "count": 213, "max_ms": 97.967, "p95_ms": 92.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.871, "count": 214, "max_ms": 187.859, "p95_ms": 160.343}, "read_interval": {"avg_ms": 140.705, "count": 213, "max_ms": 196.256, "p95_ms": 170.96}, "triton_wait": {"avg_ms": 76.216, "count": 213, "max_ms": 117.29, "p95_ms": 99.486}, "visualize": {"avg_ms": 5.377, "count": 213, "max_ms": 9.007, "p95_ms": 6.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1229}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5971.047, "limit_mb": 8796093022207.996}, "detail": {"pid": 20516, "restart_count": 60, "start_count": 61}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 230, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:46:38+09:00", "last_error_summary": "[h264 @ 0x557dbb898a80] error while decoding MB 15 40, bytestream -9", "last_exit_at": "2026-07-06T17:46:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:46:40+09:00", "pid": 20516, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.953, "name": "ffmpeg", "pid": 20516, "rss_mb": 138.953, "state": "S (sleeping)", "threads": 106}, "restart_count": 60, "restart_count_per_hour": 20, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.275, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 147503, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:46:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.67, "last_frame_pushed_at": "2026-07-06T17:46:38+09:00", "last_frame_read_age_sec": 4.792, "last_frame_read_at": "2026-07-06T17:46:37+09:00", "mjpeg": {"age_sec": 4.653, "exists": true, "mtime": "2026-07-06T17:46:38.101024+09:00", "mtime_epoch": 1783327598.1010244, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183698}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:46:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1608.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.024, "pushed": 6.024, "read": 5.991}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.32, "count": 180, "max_ms": 9.762, "p95_ms": 8.213}, "gstreamer_push": {"avg_ms": 1.127, "count": 180, "max_ms": 1.954, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.85, "count": 180, "max_ms": 82.221, "p95_ms": 76.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.951, "count": 180, "max_ms": 117.276, "p95_ms": 99.474}, "yolo11_640_fire_11class": {"avg_ms": 59.068, "count": 180, "max_ms": 83.996, "p95_ms": 78.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.012, "count": 180, "max_ms": 97.967, "p95_ms": 92.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.988, "count": 180, "max_ms": 187.859, "p95_ms": 160.777}, "read_interval": {"avg_ms": 140.871, "count": 179, "max_ms": 196.256, "p95_ms": 173.408}, "triton_wait": {"avg_ms": 76.119, "count": 180, "max_ms": 117.29, "p95_ms": 99.486}, "visualize": {"avg_ms": 5.379, "count": 180, "max_ms": 9.007, "p95_ms": 6.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5874.719, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5642b0b9b640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5642b0b9b640] error while decoding MB 0 21, bytestream 3529 [h264 @ 0x5642b1694f40] error while decoding MB 6 21, bytestream -13 [h264 @ 0x5642b0bc39c0] error while decoding MB 69 34, bytestream -6 [h264 @ 0x5642b14f7240] error while decoding MB 33 31, bytestream -8 [in#0/rtsp @ 0x5642b0a66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5642b0a66e00] Error retrieving a packet from demuxer: Con", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 234, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:49:31+09:00", "last_error_summary": "[h264 @ 0x5642b0b9b640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5642b0b9b640] error while decoding MB 0 21, bytestream 3529 [h264 @ 0x5642b1694f40] error while decoding MB 6 21, bytestream -13 [h264 @ 0x5642b0bc39c0] error while decoding MB 69 34, bytestream -6 [h264 @ 0x5642b14f7240] error while decoding MB 33 31, bytestream -8 [in#0/rtsp @ 0x5642b0a66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5642b0a66e00] Error retrieving a packet from demuxer: Con", "last_exit_at": "2026-07-06T17:46:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 170.738, "last_started_at": "2026-07-06T17:46:40+09:00", "pid": 20516, "process": {"exists": false, "pid": 20516}, "restart_count": 60, "restart_count_per_hour": 19, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.88, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 148552, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:49:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T17:49:31+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T17:49:31+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T17:49:31.451743+09:00", "mtime_epoch": 1783327771.4517434, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185181}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:49:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1609.188, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.438, "pushed": 2.438, "read": 2.438}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.243, "count": 73, "max_ms": 7.963, "p95_ms": 7.88}, "gstreamer_push": {"avg_ms": 1.096, "count": 73, "max_ms": 1.219, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.964, "count": 73, "max_ms": 77.382, "p95_ms": 74.019}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.455, "count": 73, "max_ms": 108.421, "p95_ms": 98.283}, "yolo11_640_fire_11class": {"avg_ms": 59.23, "count": 73, "max_ms": 80.029, "p95_ms": 76.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.625, "count": 73, "max_ms": 92.673, "p95_ms": 88.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.987, "count": 73, "max_ms": 162.543, "p95_ms": 155.321}, "read_interval": {"avg_ms": 410.998, "count": 73, "max_ms": 10126.293, "p95_ms": 170.391}, "triton_wait": {"avg_ms": 74.609, "count": 73, "max_ms": 108.433, "p95_ms": 98.297}, "visualize": {"avg_ms": 5.239, "count": 73, "max_ms": 6.545, "p95_ms": 6.252}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5874.195, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:49:31+09:00", "last_error_summary": "[h264 @ 0x5642b0b9b640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5642b0b9b640] error while decoding MB 0 21, bytestream 3529 [h264 @ 0x5642b1694f40] error while decoding MB 6 21, bytestream -13 [h264 @ 0x5642b0bc39c0] error while decoding MB 69 34, bytestream -6 [h264 @ 0x5642b14f7240] error while decoding MB 33 31, bytestream -8 [in#0/rtsp @ 0x5642b0a66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5642b0a66e00] Error retrieving a packet from demuxer: Con", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 234, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:49:31+09:00", "last_error_summary": "[h264 @ 0x5642b0b9b640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5642b0b9b640] error while decoding MB 0 21, bytestream 3529 [h264 @ 0x5642b1694f40] error while decoding MB 6 21, bytestream -13 [h264 @ 0x5642b0bc39c0] error while decoding MB 69 34, bytestream -6 [h264 @ 0x5642b14f7240] error while decoding MB 33 31, bytestream -8 [in#0/rtsp @ 0x5642b0a66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5642b0a66e00] Error retrieving a packet from demuxer: Con", "last_exit_at": "2026-07-06T17:49:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 170.809, "last_started_at": "2026-07-06T17:46:40+09:00", "pid": 20516, "process": {"exists": false, "pid": 20516}, "restart_count": 60, "restart_count_per_hour": 19, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.88, "sample_count": 1963, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 148553, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:49:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T17:49:31+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T17:49:31+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T17:49:31.451743+09:00", "mtime_epoch": 1783327771.4517434, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185181}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:49:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1609.188, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.442, "pushed": 2.442, "read": 2.409}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.259, "count": 73, "max_ms": 7.963, "p95_ms": 7.88}, "gstreamer_push": {"avg_ms": 1.096, "count": 73, "max_ms": 1.219, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.99, "count": 73, "max_ms": 77.382, "p95_ms": 74.019}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.519, "count": 73, "max_ms": 108.421, "p95_ms": 98.283}, "yolo11_640_fire_11class": {"avg_ms": 59.327, "count": 73, "max_ms": 80.029, "p95_ms": 76.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.681, "count": 73, "max_ms": 92.673, "p95_ms": 88.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.057, "count": 73, "max_ms": 162.543, "p95_ms": 155.321}, "read_interval": {"avg_ms": 414.996, "count": 72, "max_ms": 10126.293, "p95_ms": 170.391}, "triton_wait": {"avg_ms": 74.673, "count": 73, "max_ms": 108.433, "p95_ms": 98.297}, "visualize": {"avg_ms": 5.243, "count": 73, "max_ms": 6.545, "p95_ms": 6.252}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5874.184, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 234, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:49:31+09:00", "last_error_summary": "[h264 @ 0x5642b0b9b640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5642b0b9b640] error while decoding MB 0 21, bytestream 3529 [h264 @ 0x5642b1694f40] error while decoding MB 6 21, bytestream -13 [h264 @ 0x5642b0bc39c0] error while decoding MB 69 34, bytestream -6 [h264 @ 0x5642b14f7240] error while decoding MB 33 31, bytestream -8 [in#0/rtsp @ 0x5642b0a66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5642b0a66e00] Error retrieving a packet from demuxer: Con", "last_exit_at": "2026-07-06T17:49:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 170.814, "last_started_at": "2026-07-06T17:46:40+09:00", "pid": 20516, "process": {"exists": false, "pid": 20516}, "restart_count": 60, "restart_count_per_hour": 19, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.88, "sample_count": 1963, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 148553, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:49:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T17:49:31+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-06T17:49:31+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T17:49:31.451743+09:00", "mtime_epoch": 1783327771.4517434, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185181}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:49:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1609.188, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.442, "pushed": 2.442, "read": 2.408}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.259, "count": 73, "max_ms": 7.963, "p95_ms": 7.88}, "gstreamer_push": {"avg_ms": 1.096, "count": 73, "max_ms": 1.219, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.99, "count": 73, "max_ms": 77.382, "p95_ms": 74.019}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.519, "count": 73, "max_ms": 108.421, "p95_ms": 98.283}, "yolo11_640_fire_11class": {"avg_ms": 59.327, "count": 73, "max_ms": 80.029, "p95_ms": 76.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.681, "count": 73, "max_ms": 92.673, "p95_ms": 88.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.057, "count": 73, "max_ms": 162.543, "p95_ms": 155.321}, "read_interval": {"avg_ms": 414.996, "count": 72, "max_ms": 10126.293, "p95_ms": 170.391}, "triton_wait": {"avg_ms": 74.673, "count": 73, "max_ms": 108.433, "p95_ms": 98.297}, "visualize": {"avg_ms": 5.243, "count": 73, "max_ms": 6.545, "p95_ms": 6.252}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5984.848, "limit_mb": 8796093022207.996}, "detail": {"pid": 20623, "restart_count": 61, "start_count": 62}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 234, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:49:31+09:00", "last_error_summary": "[h264 @ 0x5642b0b9b640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5642b0b9b640] error while decoding MB 0 21, bytestream 3529 [h264 @ 0x5642b1694f40] error while decoding MB 6 21, bytestream -13 [h264 @ 0x5642b0bc39c0] error while decoding MB 69 34, bytestream -6 [h264 @ 0x5642b14f7240] error while decoding MB 33 31, bytestream -8 [in#0/rtsp @ 0x5642b0a66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5642b0a66e00] Error retrieving a packet from demuxer: Con", "last_exit_at": "2026-07-06T17:49:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:49:33+09:00", "pid": 20623, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.953, "name": "ffmpeg", "pid": 20623, "rss_mb": 138.953, "state": "S (sleeping)", "threads": 106}, "restart_count": 61, "restart_count_per_hour": 20, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.88, "sample_count": 1932, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 148553, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:49:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.12, "last_frame_pushed_at": "2026-07-06T17:49:31+09:00", "last_frame_read_age_sec": 4.241, "last_frame_read_at": "2026-07-06T17:49:31+09:00", "mjpeg": {"age_sec": 4.103, "exists": true, "mtime": "2026-07-06T17:49:31.575745+09:00", "mtime_epoch": 1783327771.575745, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185449}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:49:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1609.188, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.44, "pushed": 1.44, "read": 1.406}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.432, "count": 43, "max_ms": 7.963, "p95_ms": 7.911}, "gstreamer_push": {"avg_ms": 1.086, "count": 43, "max_ms": 1.219, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.197, "count": 43, "max_ms": 77.382, "p95_ms": 75.233}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.094, "count": 43, "max_ms": 108.421, "p95_ms": 97.682}, "yolo11_640_fire_11class": {"avg_ms": 59.647, "count": 43, "max_ms": 80.029, "p95_ms": 76.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.868, "count": 43, "max_ms": 92.673, "p95_ms": 90.173}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.556, "count": 43, "max_ms": 162.543, "p95_ms": 156.994}, "read_interval": {"avg_ms": 613.946, "count": 42, "max_ms": 10126.293, "p95_ms": 174.786}, "triton_wait": {"avg_ms": 75.325, "count": 43, "max_ms": 108.433, "p95_ms": 97.695}, "visualize": {"avg_ms": 5.341, "count": 43, "max_ms": 6.545, "p95_ms": 6.255}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5870.715, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562d59740c40] top block unavailable for requested intra mode -1 [h264 @ 0x562d59740c40] error while decoding MB 14 0, bytestream 5161 [h264 @ 0x562d59b4b400] left block unavailable for requested intra mode [h264 @ 0x562d59b4b400] error while decoding MB 0 25, bytestream 2716 [h264 @ 0x562d58e1e500] left block unavailable for requested intra mode [h264 @ 0x562d58e1e500] error while decoding MB 0 32, bytestream 3764 [h264 @ 0x562d58dc9800] left block unavailable for requested intra4x4 mo", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 242, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:53:28+09:00", "last_error_summary": "[h264 @ 0x562d59740c40] top block unavailable for requested intra mode -1 [h264 @ 0x562d59740c40] error while decoding MB 14 0, bytestream 5161 [h264 @ 0x562d59b4b400] left block unavailable for requested intra mode [h264 @ 0x562d59b4b400] error while decoding MB 0 25, bytestream 2716 [h264 @ 0x562d58e1e500] left block unavailable for requested intra mode [h264 @ 0x562d58e1e500] error while decoding MB 0 32, bytestream 3764 [h264 @ 0x562d58dc9800] left block unavailable for requested intra4x4 mo", "last_exit_at": "2026-07-06T17:49:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 234.804, "last_started_at": "2026-07-06T17:49:33+09:00", "pid": 20623, "process": {"exists": false, "pid": 20623}, "restart_count": 61, "restart_count_per_hour": 19, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.453, "sample_count": 1856, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 150099, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:53:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-06T17:53:28+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T17:53:28+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-06T17:53:28.435472+09:00", "mtime_epoch": 1783328008.4354722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:53:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1609.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 78, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.606, "pushed": 2.606, "read": 2.606}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.504, "count": 78, "max_ms": 9.121, "p95_ms": 8.57}, "gstreamer_push": {"avg_ms": 1.124, "count": 78, "max_ms": 1.324, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.888, "count": 78, "max_ms": 84.174, "p95_ms": 76.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.171, "count": 78, "max_ms": 117.013, "p95_ms": 103.448}, "yolo11_640_fire_11class": {"avg_ms": 62.198, "count": 78, "max_ms": 90.026, "p95_ms": 82.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.247, "count": 78, "max_ms": 106.577, "p95_ms": 96.271}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.289, "count": 78, "max_ms": 198.525, "p95_ms": 171.66}, "read_interval": {"avg_ms": 384.745, "count": 78, "max_ms": 18843.605, "p95_ms": 188.001}, "triton_wait": {"avg_ms": 78.352, "count": 78, "max_ms": 117.025, "p95_ms": 103.461}, "visualize": {"avg_ms": 5.402, "count": 78, "max_ms": 8.352, "p95_ms": 7.225}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5869.629, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:53:28+09:00", "last_error_summary": "[h264 @ 0x562d59740c40] top block unavailable for requested intra mode -1 [h264 @ 0x562d59740c40] error while decoding MB 14 0, bytestream 5161 [h264 @ 0x562d59b4b400] left block unavailable for requested intra mode [h264 @ 0x562d59b4b400] error while decoding MB 0 25, bytestream 2716 [h264 @ 0x562d58e1e500] left block unavailable for requested intra mode [h264 @ 0x562d58e1e500] error while decoding MB 0 32, bytestream 3764 [h264 @ 0x562d58dc9800] left block unavailable for requested intra4x4 mo", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 242, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:53:28+09:00", "last_error_summary": "[h264 @ 0x562d59740c40] top block unavailable for requested intra mode -1 [h264 @ 0x562d59740c40] error while decoding MB 14 0, bytestream 5161 [h264 @ 0x562d59b4b400] left block unavailable for requested intra mode [h264 @ 0x562d59b4b400] error while decoding MB 0 25, bytestream 2716 [h264 @ 0x562d58e1e500] left block unavailable for requested intra mode [h264 @ 0x562d58e1e500] error while decoding MB 0 32, bytestream 3764 [h264 @ 0x562d58dc9800] left block unavailable for requested intra4x4 mo", "last_exit_at": "2026-07-06T17:53:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 234.881, "last_started_at": "2026-07-06T17:49:33+09:00", "pid": 20623, "process": {"exists": false, "pid": 20623}, "restart_count": 61, "restart_count_per_hour": 19, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.453, "sample_count": 1857, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 150100, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:53:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T17:53:28+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T17:53:28+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T17:53:28.435472+09:00", "mtime_epoch": 1783328008.4354722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:53:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1609.961, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 78, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.6, "pushed": 2.6, "read": 2.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.505, "count": 78, "max_ms": 9.121, "p95_ms": 8.57}, "gstreamer_push": {"avg_ms": 1.126, "count": 78, "max_ms": 1.324, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.76, "count": 78, "max_ms": 84.174, "p95_ms": 76.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.062, "count": 78, "max_ms": 117.013, "p95_ms": 103.448}, "yolo11_640_fire_11class": {"avg_ms": 62.095, "count": 78, "max_ms": 90.026, "p95_ms": 82.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.138, "count": 78, "max_ms": 106.577, "p95_ms": 96.271}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.185, "count": 78, "max_ms": 198.525, "p95_ms": 171.66}, "read_interval": {"avg_ms": 384.745, "count": 78, "max_ms": 18843.605, "p95_ms": 188.001}, "triton_wait": {"avg_ms": 78.243, "count": 78, "max_ms": 117.025, "p95_ms": 103.461}, "visualize": {"avg_ms": 5.404, "count": 78, "max_ms": 8.352, "p95_ms": 7.225}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5869.617, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 242, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:53:28+09:00", "last_error_summary": "[h264 @ 0x562d59740c40] top block unavailable for requested intra mode -1 [h264 @ 0x562d59740c40] error while decoding MB 14 0, bytestream 5161 [h264 @ 0x562d59b4b400] left block unavailable for requested intra mode [h264 @ 0x562d59b4b400] error while decoding MB 0 25, bytestream 2716 [h264 @ 0x562d58e1e500] left block unavailable for requested intra mode [h264 @ 0x562d58e1e500] error while decoding MB 0 32, bytestream 3764 [h264 @ 0x562d58dc9800] left block unavailable for requested intra4x4 mo", "last_exit_at": "2026-07-06T17:53:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 234.886, "last_started_at": "2026-07-06T17:49:33+09:00", "pid": 20623, "process": {"exists": false, "pid": 20623}, "restart_count": 61, "restart_count_per_hour": 19, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.453, "sample_count": 1857, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 150100, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:53:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T17:53:28+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T17:53:28+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-06T17:53:28.435472+09:00", "mtime_epoch": 1783328008.4354722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:53:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1609.961, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 77, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.61, "pushed": 2.61, "read": 2.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.505, "count": 78, "max_ms": 9.121, "p95_ms": 8.57}, "gstreamer_push": {"avg_ms": 1.126, "count": 78, "max_ms": 1.324, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.76, "count": 78, "max_ms": 84.174, "p95_ms": 76.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.062, "count": 78, "max_ms": 117.013, "p95_ms": 103.448}, "yolo11_640_fire_11class": {"avg_ms": 62.095, "count": 78, "max_ms": 90.026, "p95_ms": 82.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.138, "count": 78, "max_ms": 106.577, "p95_ms": 96.271}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.185, "count": 78, "max_ms": 198.525, "p95_ms": 171.66}, "read_interval": {"avg_ms": 387.897, "count": 77, "max_ms": 18843.605, "p95_ms": 188.001}, "triton_wait": {"avg_ms": 78.243, "count": 78, "max_ms": 117.025, "p95_ms": 103.461}, "visualize": {"avg_ms": 5.404, "count": 78, "max_ms": 8.352, "p95_ms": 7.225}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.914, "limit_mb": 8796093022207.996}, "detail": {"pid": 20730, "restart_count": 62, "start_count": 63}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 242, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:53:28+09:00", "last_error_summary": "[h264 @ 0x562d59740c40] top block unavailable for requested intra mode -1 [h264 @ 0x562d59740c40] error while decoding MB 14 0, bytestream 5161 [h264 @ 0x562d59b4b400] left block unavailable for requested intra mode [h264 @ 0x562d59b4b400] error while decoding MB 0 25, bytestream 2716 [h264 @ 0x562d58e1e500] left block unavailable for requested intra mode [h264 @ 0x562d58e1e500] error while decoding MB 0 32, bytestream 3764 [h264 @ 0x562d58dc9800] left block unavailable for requested intra4x4 mo", "last_exit_at": "2026-07-06T17:53:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:53:30+09:00", "pid": 20730, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.105, "name": "ffmpeg", "pid": 20730, "rss_mb": 139.105, "state": "S (sleeping)", "threads": 106}, "restart_count": 62, "restart_count_per_hour": 20, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.453, "sample_count": 1827, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 150100, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:53:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.158, "last_frame_pushed_at": "2026-07-06T17:53:28+09:00", "last_frame_read_age_sec": 4.286, "last_frame_read_at": "2026-07-06T17:53:28+09:00", "mjpeg": {"age_sec": 4.141, "exists": true, "mtime": "2026-07-06T17:53:28.571475+09:00", "mtime_epoch": 1783328008.5714746, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178036}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:53:32+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1609.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.67, "pushed": 1.67, "read": 1.637}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.329, "count": 50, "max_ms": 8.047, "p95_ms": 7.452}, "gstreamer_push": {"avg_ms": 1.13, "count": 50, "max_ms": 1.324, "p95_ms": 1.223}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.604, "count": 50, "max_ms": 84.174, "p95_ms": 76.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.624, "count": 50, "max_ms": 117.013, "p95_ms": 100.768}, "yolo11_640_fire_11class": {"avg_ms": 62.158, "count": 50, "max_ms": 90.026, "p95_ms": 82.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.045, "count": 50, "max_ms": 106.577, "p95_ms": 97.144}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.176, "count": 50, "max_ms": 198.525, "p95_ms": 169.753}, "read_interval": {"avg_ms": 525.806, "count": 49, "max_ms": 18843.605, "p95_ms": 188.001}, "triton_wait": {"avg_ms": 78.756, "count": 50, "max_ms": 117.025, "p95_ms": 100.781}, "visualize": {"avg_ms": 5.231, "count": 50, "max_ms": 6.569, "p95_ms": 6.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1263}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5882.172, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558adb1a6b00] reference picture missing during reorder [h264 @ 0x558adb1a6b00] Missing reference picture, default is 65580 [h264 @ 0x558adb1c21c0] mmco: unref short failure [h264 @ 0x558adb1c21c0] illegal short term buffer state detected [h264 @ 0x558adb2ddf00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558adb2ddf00] error while decoding MB 0 44, bytestream 4179 [h264 @ 0x558adb2f20c0] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558adb2f20c0] error while ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 248, "h264_decode_error_count_per_hour": 54, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:57:26+09:00", "last_error_summary": "[h264 @ 0x558adb1a6b00] reference picture missing during reorder [h264 @ 0x558adb1a6b00] Missing reference picture, default is 65580 [h264 @ 0x558adb1c21c0] mmco: unref short failure [h264 @ 0x558adb1c21c0] illegal short term buffer state detected [h264 @ 0x558adb2ddf00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558adb2ddf00] error while decoding MB 0 44, bytestream 4179 [h264 @ 0x558adb2f20c0] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558adb2f20c0] error while ", "last_exit_at": "2026-07-06T17:53:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 235.412, "last_started_at": "2026-07-06T17:53:30+09:00", "pid": 20730, "process": {"exists": false, "pid": 20730}, "restart_count": 62, "restart_count_per_hour": 18, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.526, "sample_count": 1854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 151644, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:57:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T17:57:26+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T17:57:26+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T17:57:26.083221+09:00", "mtime_epoch": 1783328246.0832214, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178614}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:57:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1610.734, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.772, "pushed": 2.772, "read": 2.772}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.002, "count": 83, "max_ms": 8.24, "p95_ms": 7.519}, "gstreamer_push": {"avg_ms": 1.104, "count": 83, "max_ms": 1.361, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.081, "count": 83, "max_ms": 84.169, "p95_ms": 75.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.589, "count": 83, "max_ms": 121.644, "p95_ms": 97.583}, "yolo11_640_fire_11class": {"avg_ms": 59.701, "count": 83, "max_ms": 85.946, "p95_ms": 79.295}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.484, "count": 83, "max_ms": 98.658, "p95_ms": 91.184}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.587, "count": 83, "max_ms": 175.744, "p95_ms": 151.558}, "read_interval": {"avg_ms": 361.916, "count": 83, "max_ms": 10128.127, "p95_ms": 163.649}, "triton_wait": {"avg_ms": 74.667, "count": 83, "max_ms": 121.657, "p95_ms": 97.597}, "visualize": {"avg_ms": 5.104, "count": 83, "max_ms": 6.504, "p95_ms": 5.75}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1263}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5881.621, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T17:57:26+09:00", "last_error_summary": "[h264 @ 0x558adb1a6b00] reference picture missing during reorder [h264 @ 0x558adb1a6b00] Missing reference picture, default is 65580 [h264 @ 0x558adb1c21c0] mmco: unref short failure [h264 @ 0x558adb1c21c0] illegal short term buffer state detected [h264 @ 0x558adb2ddf00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558adb2ddf00] error while decoding MB 0 44, bytestream 4179 [h264 @ 0x558adb2f20c0] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558adb2f20c0] error while ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 248, "h264_decode_error_count_per_hour": 54, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:57:26+09:00", "last_error_summary": "[h264 @ 0x558adb1a6b00] reference picture missing during reorder [h264 @ 0x558adb1a6b00] Missing reference picture, default is 65580 [h264 @ 0x558adb1c21c0] mmco: unref short failure [h264 @ 0x558adb1c21c0] illegal short term buffer state detected [h264 @ 0x558adb2ddf00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558adb2ddf00] error while decoding MB 0 44, bytestream 4179 [h264 @ 0x558adb2f20c0] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558adb2f20c0] error while ", "last_exit_at": "2026-07-06T17:57:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 235.493, "last_started_at": "2026-07-06T17:53:30+09:00", "pid": 20730, "process": {"exists": false, "pid": 20730}, "restart_count": 62, "restart_count_per_hour": 18, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.526, "sample_count": 1855, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 151645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:57:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T17:57:26+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T17:57:26+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T17:57:26.083221+09:00", "mtime_epoch": 1783328246.0832214, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178614}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:57:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1610.734, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.776, "pushed": 2.776, "read": 2.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.002, "count": 83, "max_ms": 8.24, "p95_ms": 7.519}, "gstreamer_push": {"avg_ms": 1.103, "count": 83, "max_ms": 1.361, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.988, "count": 83, "max_ms": 84.169, "p95_ms": 75.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.458, "count": 83, "max_ms": 121.644, "p95_ms": 97.583}, "yolo11_640_fire_11class": {"avg_ms": 59.814, "count": 83, "max_ms": 85.946, "p95_ms": 79.295}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.564, "count": 83, "max_ms": 98.658, "p95_ms": 91.184}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.313, "count": 83, "max_ms": 175.744, "p95_ms": 151.558}, "read_interval": {"avg_ms": 364.351, "count": 82, "max_ms": 10128.127, "p95_ms": 163.649}, "triton_wait": {"avg_ms": 74.536, "count": 83, "max_ms": 121.657, "p95_ms": 97.597}, "visualize": {"avg_ms": 5.101, "count": 83, "max_ms": 6.504, "p95_ms": 5.75}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1263}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5881.676, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 248, "h264_decode_error_count_per_hour": 54, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:57:26+09:00", "last_error_summary": "[h264 @ 0x558adb1a6b00] reference picture missing during reorder [h264 @ 0x558adb1a6b00] Missing reference picture, default is 65580 [h264 @ 0x558adb1c21c0] mmco: unref short failure [h264 @ 0x558adb1c21c0] illegal short term buffer state detected [h264 @ 0x558adb2ddf00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558adb2ddf00] error while decoding MB 0 44, bytestream 4179 [h264 @ 0x558adb2f20c0] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558adb2f20c0] error while ", "last_exit_at": "2026-07-06T17:57:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 235.497, "last_started_at": "2026-07-06T17:53:30+09:00", "pid": 20730, "process": {"exists": false, "pid": 20730}, "restart_count": 62, "restart_count_per_hour": 18, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.526, "sample_count": 1855, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 151645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:57:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T17:57:26+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-06T17:57:26+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-06T17:57:26.083221+09:00", "mtime_epoch": 1783328246.0832214, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178614}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:57:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1610.734, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.775, "pushed": 2.775, "read": 2.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.002, "count": 83, "max_ms": 8.24, "p95_ms": 7.519}, "gstreamer_push": {"avg_ms": 1.103, "count": 83, "max_ms": 1.361, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.988, "count": 83, "max_ms": 84.169, "p95_ms": 75.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.458, "count": 83, "max_ms": 121.644, "p95_ms": 97.583}, "yolo11_640_fire_11class": {"avg_ms": 59.814, "count": 83, "max_ms": 85.946, "p95_ms": 79.295}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.564, "count": 83, "max_ms": 98.658, "p95_ms": 91.184}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.313, "count": 83, "max_ms": 175.744, "p95_ms": 151.558}, "read_interval": {"avg_ms": 364.351, "count": 82, "max_ms": 10128.127, "p95_ms": 163.649}, "triton_wait": {"avg_ms": 74.536, "count": 83, "max_ms": 121.657, "p95_ms": 97.597}, "visualize": {"avg_ms": 5.101, "count": 83, "max_ms": 6.504, "p95_ms": 5.75}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1263}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5984.051, "limit_mb": 8796093022207.996}, "detail": {"pid": 20837, "restart_count": 63, "start_count": 64}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 248, "h264_decode_error_count_per_hour": 54, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T17:57:26+09:00", "last_error_summary": "[h264 @ 0x558adb1a6b00] reference picture missing during reorder [h264 @ 0x558adb1a6b00] Missing reference picture, default is 65580 [h264 @ 0x558adb1c21c0] mmco: unref short failure [h264 @ 0x558adb1c21c0] illegal short term buffer state detected [h264 @ 0x558adb2ddf00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558adb2ddf00] error while decoding MB 0 44, bytestream 4179 [h264 @ 0x558adb2f20c0] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558adb2f20c0] error while ", "last_exit_at": "2026-07-06T17:57:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T17:57:28+09:00", "pid": 20837, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.188, "name": "ffmpeg", "pid": 20837, "rss_mb": 139.188, "state": "S (sleeping)", "threads": 106}, "restart_count": 63, "restart_count_per_hour": 19, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.633, "sample_count": 1822, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 151645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T17:57:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.577, "last_frame_pushed_at": "2026-07-06T17:57:26+09:00", "last_frame_read_age_sec": 4.706, "last_frame_read_at": "2026-07-06T17:57:26+09:00", "mjpeg": {"age_sec": 4.56, "exists": true, "mtime": "2026-07-06T17:57:26.215223+09:00", "mtime_epoch": 1783328246.2152233, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178812}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T17:57:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1610.734, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.668, "pushed": 1.668, "read": 1.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.971, "count": 49, "max_ms": 8.24, "p95_ms": 7.853}, "gstreamer_push": {"avg_ms": 1.099, "count": 50, "max_ms": 1.361, "p95_ms": 1.274}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.039, "count": 49, "max_ms": 84.169, "p95_ms": 75.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.733, "count": 49, "max_ms": 121.644, "p95_ms": 98.534}, "yolo11_640_fire_11class": {"avg_ms": 59.303, "count": 49, "max_ms": 85.946, "p95_ms": 79.295}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.587, "count": 49, "max_ms": 98.658, "p95_ms": 91.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.742, "count": 50, "max_ms": 175.744, "p95_ms": 152.064}, "read_interval": {"avg_ms": 518.042, "count": 49, "max_ms": 10128.127, "p95_ms": 185.082}, "triton_wait": {"avg_ms": 74.791, "count": 49, "max_ms": 121.657, "p95_ms": 98.546}, "visualize": {"avg_ms": 5.132, "count": 49, "max_ms": 6.504, "p95_ms": 6.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1276}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5873.293, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:00:57+09:00", "last_error_summary": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "last_exit_at": "2026-07-06T17:57:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 208.894, "last_started_at": "2026-07-06T17:57:28+09:00", "pid": 20837, "process": {"exists": false, "pid": 20837}, "restart_count": 63, "restart_count_per_hour": 18, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.15, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 153147, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:00:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T18:00:57+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T18:00:57+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-06T18:00:57.622564+09:00", "mtime_epoch": 1783328457.6225636, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:00:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1611.25, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.261, "pushed": 7.261, "read": 7.261}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.178, "count": 217, "max_ms": 9.42, "p95_ms": 7.849}, "gstreamer_push": {"avg_ms": 1.109, "count": 217, "max_ms": 1.534, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.826, "count": 217, "max_ms": 83.737, "p95_ms": 77.625}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.625, "count": 217, "max_ms": 125.399, "p95_ms": 98.582}, "yolo11_640_fire_11class": {"avg_ms": 61.975, "count": 217, "max_ms": 90.795, "p95_ms": 83.198}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.248, "count": 217, "max_ms": 98.381, "p95_ms": 91.667}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.449, "count": 217, "max_ms": 179.335, "p95_ms": 152.646}, "read_interval": {"avg_ms": 137.996, "count": 217, "max_ms": 192.14, "p95_ms": 163.264}, "triton_wait": {"avg_ms": 74.903, "count": 217, "max_ms": 125.413, "p95_ms": 98.592}, "visualize": {"avg_ms": 5.257, "count": 217, "max_ms": 7.577, "p95_ms": 6.234}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1276}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5872.566, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:00:57+09:00", "last_error_summary": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:00:57+09:00", "last_error_summary": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "last_exit_at": "2026-07-06T18:00:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 208.972, "last_started_at": "2026-07-06T17:57:28+09:00", "pid": 20837, "process": {"exists": false, "pid": 20837}, "restart_count": 63, "restart_count_per_hour": 18, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.15, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 153148, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:00:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T18:00:57+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T18:00:57+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-06T18:00:57.622564+09:00", "mtime_epoch": 1783328457.6225636, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:00:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1611.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.275, "pushed": 7.275, "read": 7.242}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.202, "count": 217, "max_ms": 11.256, "p95_ms": 7.89}, "gstreamer_push": {"avg_ms": 1.109, "count": 218, "max_ms": 1.534, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.79, "count": 217, "max_ms": 83.737, "p95_ms": 77.625}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.611, "count": 217, "max_ms": 125.399, "p95_ms": 98.582}, "yolo11_640_fire_11class": {"avg_ms": 61.943, "count": 217, "max_ms": 90.795, "p95_ms": 83.198}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.224, "count": 217, "max_ms": 98.381, "p95_ms": 91.667}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.422, "count": 218, "max_ms": 179.335, "p95_ms": 152.646}, "read_interval": {"avg_ms": 137.996, "count": 217, "max_ms": 192.14, "p95_ms": 163.264}, "triton_wait": {"avg_ms": 74.889, "count": 217, "max_ms": 125.413, "p95_ms": 98.592}, "visualize": {"avg_ms": 5.263, "count": 217, "max_ms": 7.577, "p95_ms": 6.332}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1276}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5872.559, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:00:57+09:00", "last_error_summary": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "last_exit_at": "2026-07-06T18:00:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 208.978, "last_started_at": "2026-07-06T17:57:28+09:00", "pid": 20837, "process": {"exists": false, "pid": 20837}, "restart_count": 63, "restart_count_per_hour": 18, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.15, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 153148, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:00:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T18:00:57+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T18:00:57+09:00", "mjpeg": {"age_sec": 0.129, "exists": true, "mtime": "2026-07-06T18:00:57.622564+09:00", "mtime_epoch": 1783328457.6225636, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:00:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1611.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.274, "pushed": 7.274, "read": 7.241}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.202, "count": 217, "max_ms": 11.256, "p95_ms": 7.89}, "gstreamer_push": {"avg_ms": 1.109, "count": 218, "max_ms": 1.534, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.79, "count": 217, "max_ms": 83.737, "p95_ms": 77.625}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.611, "count": 217, "max_ms": 125.399, "p95_ms": 98.582}, "yolo11_640_fire_11class": {"avg_ms": 61.943, "count": 217, "max_ms": 90.795, "p95_ms": 83.198}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.224, "count": 217, "max_ms": 98.381, "p95_ms": 91.667}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.422, "count": 218, "max_ms": 179.335, "p95_ms": 152.646}, "read_interval": {"avg_ms": 137.996, "count": 217, "max_ms": 192.14, "p95_ms": 163.264}, "triton_wait": {"avg_ms": 74.889, "count": 217, "max_ms": 125.413, "p95_ms": 98.592}, "visualize": {"avg_ms": 5.263, "count": 217, "max_ms": 7.577, "p95_ms": 6.332}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1276}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.914, "limit_mb": 8796093022207.996}, "detail": {"pid": 20944, "restart_count": 64, "start_count": 65}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:00:57+09:00", "last_error_summary": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "last_exit_at": "2026-07-06T18:01:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:01:00+09:00", "pid": 20944, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.996, "name": "ffmpeg", "pid": 20944, "rss_mb": 141.996, "state": "S (sleeping)", "threads": 106}, "restart_count": 64, "restart_count_per_hour": 19, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.137, "sample_count": 1972, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 153148, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:00:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.855, "last_frame_pushed_at": "2026-07-06T18:00:57+09:00", "last_frame_read_age_sec": 4.98, "last_frame_read_at": "2026-07-06T18:00:57+09:00", "mjpeg": {"age_sec": 4.838, "exists": true, "mtime": "2026-07-06T18:00:57.754565+09:00", "mtime_epoch": 1783328457.7545655, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166183}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:01:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1611.25, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.138, "pushed": 6.138, "read": 6.104}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.196, "count": 183, "max_ms": 11.256, "p95_ms": 7.849}, "gstreamer_push": {"avg_ms": 1.107, "count": 184, "max_ms": 1.534, "p95_ms": 1.235}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.424, "count": 183, "max_ms": 83.737, "p95_ms": 76.57}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.991, "count": 183, "max_ms": 125.399, "p95_ms": 98.841}, "yolo11_640_fire_11class": {"avg_ms": 61.801, "count": 183, "max_ms": 90.795, "p95_ms": 83.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.901, "count": 183, "max_ms": 98.381, "p95_ms": 91.435}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.721, "count": 184, "max_ms": 179.335, "p95_ms": 152.516}, "read_interval": {"avg_ms": 137.27, "count": 183, "max_ms": 192.14, "p95_ms": 162.134}, "triton_wait": {"avg_ms": 74.296, "count": 183, "max_ms": 125.413, "p95_ms": 98.854}, "visualize": {"avg_ms": 5.27, "count": 183, "max_ms": 7.577, "p95_ms": 6.378}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1287}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5879.98, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:00:57+09:00", "last_error_summary": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:00:57+09:00", "last_error_summary": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "last_exit_at": "2026-07-06T18:04:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.109, "last_started_at": "2026-07-06T18:01:00+09:00", "pid": 20944, "process": {"exists": false, "pid": 20944}, "restart_count": 64, "restart_count_per_hour": 18, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.99, "sample_count": 2132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 154506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:04:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:04:10+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T18:04:10+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T18:04:10.593615+09:00", "mtime_epoch": 1783328650.593615, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177552}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:04:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1612.023, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.235, "pushed": 7.235, "read": 7.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.284, "count": 216, "max_ms": 9.731, "p95_ms": 8.13}, "gstreamer_push": {"avg_ms": 1.106, "count": 217, "max_ms": 1.553, "p95_ms": 1.229}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.393, "count": 216, "max_ms": 79.038, "p95_ms": 74.498}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.754, "count": 216, "max_ms": 117.405, "p95_ms": 100.883}, "yolo11_640_fire_11class": {"avg_ms": 60.173, "count": 216, "max_ms": 89.898, "p95_ms": 79.065}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.453, "count": 216, "max_ms": 99.973, "p95_ms": 90.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.869, "count": 217, "max_ms": 178.199, "p95_ms": 155.701}, "read_interval": {"avg_ms": 138.849, "count": 216, "max_ms": 192.735, "p95_ms": 167.292}, "triton_wait": {"avg_ms": 75.156, "count": 216, "max_ms": 117.418, "p95_ms": 100.895}, "visualize": {"avg_ms": 5.216, "count": 216, "max_ms": 7.005, "p95_ms": 6.35}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1287}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5882.738, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:00:57+09:00", "last_error_summary": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "last_exit_at": "2026-07-06T18:04:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.115, "last_started_at": "2026-07-06T18:01:00+09:00", "pid": 20944, "process": {"exists": false, "pid": 20944}, "restart_count": 64, "restart_count_per_hour": 18, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.99, "sample_count": 2132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 154506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:04:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T18:04:10+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T18:04:10+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T18:04:10.593615+09:00", "mtime_epoch": 1783328650.593615, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177552}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:04:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1612.023, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.234, "pushed": 7.234, "read": 7.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.284, "count": 216, "max_ms": 9.731, "p95_ms": 8.13}, "gstreamer_push": {"avg_ms": 1.106, "count": 217, "max_ms": 1.553, "p95_ms": 1.229}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.393, "count": 216, "max_ms": 79.038, "p95_ms": 74.498}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.754, "count": 216, "max_ms": 117.405, "p95_ms": 100.883}, "yolo11_640_fire_11class": {"avg_ms": 60.173, "count": 216, "max_ms": 89.898, "p95_ms": 79.065}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.453, "count": 216, "max_ms": 99.973, "p95_ms": 90.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.869, "count": 217, "max_ms": 178.199, "p95_ms": 155.701}, "read_interval": {"avg_ms": 138.849, "count": 216, "max_ms": 192.735, "p95_ms": 167.292}, "triton_wait": {"avg_ms": 75.156, "count": 216, "max_ms": 117.418, "p95_ms": 100.895}, "visualize": {"avg_ms": 5.216, "count": 216, "max_ms": 7.005, "p95_ms": 6.35}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1287}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5990.266, "limit_mb": 8796093022207.996}, "detail": {"pid": 21051, "restart_count": 65, "start_count": 66}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:00:57+09:00", "last_error_summary": "[h264 @ 0x55b260ac5b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b260ac5b40] error while decoding MB 0 26, bytestream 5137 [h264 @ 0x55b261539000] error while decoding MB 63 32, bytestream -5", "last_exit_at": "2026-07-06T18:04:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:04:13+09:00", "pid": 21051, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.789, "name": "ffmpeg", "pid": 21051, "rss_mb": 138.789, "state": "R (running)", "threads": 106}, "restart_count": 65, "restart_count_per_hour": 19, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.068, "sample_count": 2097, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 154506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:04:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.504, "last_frame_pushed_at": "2026-07-06T18:04:10+09:00", "last_frame_read_age_sec": 4.618, "last_frame_read_at": "2026-07-06T18:04:10+09:00", "mjpeg": {"age_sec": 4.487, "exists": true, "mtime": "2026-07-06T18:04:10.713617+09:00", "mtime_epoch": 1783328650.7136168, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178059}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:04:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1612.023, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.174, "pushed": 6.174, "read": 6.141}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.195, "count": 184, "max_ms": 9.731, "p95_ms": 7.801}, "gstreamer_push": {"avg_ms": 1.102, "count": 185, "max_ms": 1.379, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.341, "count": 184, "max_ms": 78.06, "p95_ms": 74.011}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.716, "count": 184, "max_ms": 117.405, "p95_ms": 100.796}, "yolo11_640_fire_11class": {"avg_ms": 60.39, "count": 184, "max_ms": 89.898, "p95_ms": 78.965}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.676, "count": 184, "max_ms": 99.973, "p95_ms": 90.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.417, "count": 185, "max_ms": 174.532, "p95_ms": 154.709}, "read_interval": {"avg_ms": 138.368, "count": 184, "max_ms": 182.901, "p95_ms": 165.582}, "triton_wait": {"avg_ms": 75.116, "count": 184, "max_ms": 117.418, "p95_ms": 100.81}, "visualize": {"avg_ms": 5.184, "count": 184, "max_ms": 7.005, "p95_ms": 6.163}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1299}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5890.902, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5567ca985fc0] cabac decode of qscale diff failed at 62 41 [h264 @ 0x5567ca985fc0] error while decoding MB 62 41, bytestream 115635 [h264 @ 0x5567cb464080] error while decoding MB 77 44, bytestream -7 [h264 @ 0x5567cb2c6380] cabac decode of qscale diff failed at 55 37 [h264 @ 0x5567cb2c6380] error while decoding MB 55 37, bytestream 5880 [h264 @ 0x5567ca8646c0] error while decoding MB 37 26, bytestream 205 [in#0/rtsp @ 0x5567ca849e00] Error during demuxing: Connection timed out [in#0/rt", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 256, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:07:48+09:00", "last_error_summary": "[h264 @ 0x5567ca985fc0] cabac decode of qscale diff failed at 62 41 [h264 @ 0x5567ca985fc0] error while decoding MB 62 41, bytestream 115635 [h264 @ 0x5567cb464080] error while decoding MB 77 44, bytestream -7 [h264 @ 0x5567cb2c6380] cabac decode of qscale diff failed at 55 37 [h264 @ 0x5567cb2c6380] error while decoding MB 55 37, bytestream 5880 [h264 @ 0x5567ca8646c0] error while decoding MB 37 26, bytestream 205 [in#0/rtsp @ 0x5567ca849e00] Error during demuxing: Connection timed out [in#0/rt", "last_exit_at": "2026-07-06T18:04:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 215.465, "last_started_at": "2026-07-06T18:04:13+09:00", "pid": 21051, "process": {"exists": false, "pid": 21051}, "restart_count": 65, "restart_count_per_hour": 18, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.025, "sample_count": 2001, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 155913, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:07:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T18:07:48+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T18:07:48+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T18:07:48.622179+09:00", "mtime_epoch": 1783328868.6221786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180885}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:07:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1612.539, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 81, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.703, "pushed": 2.703, "read": 2.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.134, "count": 80, "max_ms": 9.466, "p95_ms": 7.757}, "gstreamer_push": {"avg_ms": 1.102, "count": 81, "max_ms": 1.327, "p95_ms": 1.213}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.679, "count": 80, "max_ms": 84.807, "p95_ms": 80.025}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.678, "count": 80, "max_ms": 111.305, "p95_ms": 104.193}, "yolo11_640_fire_11class": {"avg_ms": 63.557, "count": 80, "max_ms": 91.025, "p95_ms": 84.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.383, "count": 80, "max_ms": 101.646, "p95_ms": 93.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.794, "count": 81, "max_ms": 164.135, "p95_ms": 161.424}, "read_interval": {"avg_ms": 370.766, "count": 81, "max_ms": 18876.604, "p95_ms": 173.018}, "triton_wait": {"avg_ms": 76.933, "count": 80, "max_ms": 111.318, "p95_ms": 104.212}, "visualize": {"avg_ms": 5.163, "count": 80, "max_ms": 8.842, "p95_ms": 6.152}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1299}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5890.062, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:07:48+09:00", "last_error_summary": "[h264 @ 0x5567ca985fc0] cabac decode of qscale diff failed at 62 41 [h264 @ 0x5567ca985fc0] error while decoding MB 62 41, bytestream 115635 [h264 @ 0x5567cb464080] error while decoding MB 77 44, bytestream -7 [h264 @ 0x5567cb2c6380] cabac decode of qscale diff failed at 55 37 [h264 @ 0x5567cb2c6380] error while decoding MB 55 37, bytestream 5880 [h264 @ 0x5567ca8646c0] error while decoding MB 37 26, bytestream 205 [in#0/rtsp @ 0x5567ca849e00] Error during demuxing: Connection timed out [in#0/rt", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 256, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:07:48+09:00", "last_error_summary": "[h264 @ 0x5567ca985fc0] cabac decode of qscale diff failed at 62 41 [h264 @ 0x5567ca985fc0] error while decoding MB 62 41, bytestream 115635 [h264 @ 0x5567cb464080] error while decoding MB 77 44, bytestream -7 [h264 @ 0x5567cb2c6380] cabac decode of qscale diff failed at 55 37 [h264 @ 0x5567cb2c6380] error while decoding MB 55 37, bytestream 5880 [h264 @ 0x5567ca8646c0] error while decoding MB 37 26, bytestream 205 [in#0/rtsp @ 0x5567ca849e00] Error during demuxing: Connection timed out [in#0/rt", "last_exit_at": "2026-07-06T18:07:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 215.562, "last_started_at": "2026-07-06T18:04:13+09:00", "pid": 21051, "process": {"exists": false, "pid": 21051}, "restart_count": 65, "restart_count_per_hour": 18, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.025, "sample_count": 2001, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 155914, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:07:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T18:07:48+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T18:07:48+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-06T18:07:48.622179+09:00", "mtime_epoch": 1783328868.6221786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180885}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:07:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1612.539, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.705, "pushed": 2.705, "read": 2.672}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.133, "count": 81, "max_ms": 9.466, "p95_ms": 7.757}, "gstreamer_push": {"avg_ms": 1.1, "count": 81, "max_ms": 1.327, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.556, "count": 81, "max_ms": 84.807, "p95_ms": 80.025}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.853, "count": 81, "max_ms": 111.305, "p95_ms": 104.193}, "yolo11_640_fire_11class": {"avg_ms": 63.739, "count": 81, "max_ms": 91.025, "p95_ms": 84.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.588, "count": 81, "max_ms": 101.646, "p95_ms": 93.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.149, "count": 81, "max_ms": 164.135, "p95_ms": 161.424}, "read_interval": {"avg_ms": 373.826, "count": 80, "max_ms": 18876.604, "p95_ms": 173.018}, "triton_wait": {"avg_ms": 77.105, "count": 81, "max_ms": 111.318, "p95_ms": 104.212}, "visualize": {"avg_ms": 5.162, "count": 81, "max_ms": 8.842, "p95_ms": 6.152}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1299}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5890.176, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 256, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:07:48+09:00", "last_error_summary": "[h264 @ 0x5567ca985fc0] cabac decode of qscale diff failed at 62 41 [h264 @ 0x5567ca985fc0] error while decoding MB 62 41, bytestream 115635 [h264 @ 0x5567cb464080] error while decoding MB 77 44, bytestream -7 [h264 @ 0x5567cb2c6380] cabac decode of qscale diff failed at 55 37 [h264 @ 0x5567cb2c6380] error while decoding MB 55 37, bytestream 5880 [h264 @ 0x5567ca8646c0] error while decoding MB 37 26, bytestream 205 [in#0/rtsp @ 0x5567ca849e00] Error during demuxing: Connection timed out [in#0/rt", "last_exit_at": "2026-07-06T18:07:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 215.567, "last_started_at": "2026-07-06T18:04:13+09:00", "pid": 21051, "process": {"exists": false, "pid": 21051}, "restart_count": 65, "restart_count_per_hour": 18, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.025, "sample_count": 2001, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 155914, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:07:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T18:07:48+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-06T18:07:48+09:00", "mjpeg": {"age_sec": 0.144, "exists": true, "mtime": "2026-07-06T18:07:48.622179+09:00", "mtime_epoch": 1783328868.6221786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180885}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:07:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1612.539, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.705, "pushed": 2.705, "read": 2.671}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.133, "count": 81, "max_ms": 9.466, "p95_ms": 7.757}, "gstreamer_push": {"avg_ms": 1.1, "count": 81, "max_ms": 1.327, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.556, "count": 81, "max_ms": 84.807, "p95_ms": 80.025}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.853, "count": 81, "max_ms": 111.305, "p95_ms": 104.193}, "yolo11_640_fire_11class": {"avg_ms": 63.739, "count": 81, "max_ms": 91.025, "p95_ms": 84.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.588, "count": 81, "max_ms": 101.646, "p95_ms": 93.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.149, "count": 81, "max_ms": 164.135, "p95_ms": 161.424}, "read_interval": {"avg_ms": 373.826, "count": 80, "max_ms": 18876.604, "p95_ms": 173.018}, "triton_wait": {"avg_ms": 77.105, "count": 81, "max_ms": 111.318, "p95_ms": 104.212}, "visualize": {"avg_ms": 5.162, "count": 81, "max_ms": 8.842, "p95_ms": 6.152}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1299}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5998.141, "limit_mb": 8796093022207.996}, "detail": {"pid": 21158, "restart_count": 66, "start_count": 67}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 256, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:07:48+09:00", "last_error_summary": "[h264 @ 0x5567ca985fc0] cabac decode of qscale diff failed at 62 41 [h264 @ 0x5567ca985fc0] error while decoding MB 62 41, bytestream 115635 [h264 @ 0x5567cb464080] error while decoding MB 77 44, bytestream -7 [h264 @ 0x5567cb2c6380] cabac decode of qscale diff failed at 55 37 [h264 @ 0x5567cb2c6380] error while decoding MB 55 37, bytestream 5880 [h264 @ 0x5567ca8646c0] error while decoding MB 37 26, bytestream 205 [in#0/rtsp @ 0x5567ca849e00] Error during demuxing: Connection timed out [in#0/rt", "last_exit_at": "2026-07-06T18:07:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:07:50+09:00", "pid": 21158, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.371, "name": "ffmpeg", "pid": 21158, "rss_mb": 141.371, "state": "S (sleeping)", "threads": 106}, "restart_count": 66, "restart_count_per_hour": 19, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.088, "sample_count": 1971, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 155914, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:07:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.208, "last_frame_pushed_at": "2026-07-06T18:07:48+09:00", "last_frame_read_age_sec": 4.355, "last_frame_read_at": "2026-07-06T18:07:48+09:00", "mjpeg": {"age_sec": 4.191, "exists": true, "mtime": "2026-07-06T18:07:48.774184+09:00", "mtime_epoch": 1783328868.7741835, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176415}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:07:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1612.539, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.701, "pushed": 1.701, "read": 1.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.166, "count": 50, "max_ms": 9.466, "p95_ms": 8.284}, "gstreamer_push": {"avg_ms": 1.096, "count": 51, "max_ms": 1.327, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.406, "count": 50, "max_ms": 84.807, "p95_ms": 82.468}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.537, "count": 50, "max_ms": 109.034, "p95_ms": 104.193}, "yolo11_640_fire_11class": {"avg_ms": 63.962, "count": 50, "max_ms": 90.595, "p95_ms": 84.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.522, "count": 50, "max_ms": 101.646, "p95_ms": 95.819}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.536, "count": 51, "max_ms": 161.985, "p95_ms": 161.424}, "read_interval": {"avg_ms": 514.841, "count": 50, "max_ms": 18876.604, "p95_ms": 173.018}, "triton_wait": {"avg_ms": 77.587, "count": 50, "max_ms": 109.046, "p95_ms": 104.212}, "visualize": {"avg_ms": 5.165, "count": 50, "max_ms": 8.842, "p95_ms": 6.216}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1310}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5888.598, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 12, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 268, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:11:14+09:00", "last_error_summary": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "last_exit_at": "2026-07-06T18:07:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 203.93, "last_started_at": "2026-07-06T18:07:50+09:00", "pid": 21158, "process": {"exists": false, "pid": 21158}, "restart_count": 66, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.639, "sample_count": 1888, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 157260, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T18:11:14+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T18:11:14+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T18:11:14.848259+09:00", "mtime_epoch": 1783329074.8482587, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:11:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1613.312, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 84, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.808, "pushed": 2.808, "read": 2.808}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.17, "count": 84, "max_ms": 8.608, "p95_ms": 7.481}, "gstreamer_push": {"avg_ms": 1.093, "count": 84, "max_ms": 1.296, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.595, "count": 84, "max_ms": 78.997, "p95_ms": 73.099}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.634, "count": 84, "max_ms": 100.893, "p95_ms": 95.222}, "yolo11_640_fire_11class": {"avg_ms": 60.5, "count": 84, "max_ms": 86.587, "p95_ms": 80.05}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.416, "count": 84, "max_ms": 95.84, "p95_ms": 90.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.55, "count": 84, "max_ms": 155.28, "p95_ms": 150.524}, "read_interval": {"avg_ms": 357.104, "count": 84, "max_ms": 10122.713, "p95_ms": 163.753}, "triton_wait": {"avg_ms": 72.375, "count": 84, "max_ms": 100.906, "p95_ms": 95.79}, "visualize": {"avg_ms": 5.204, "count": 84, "max_ms": 6.489, "p95_ms": 5.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1310}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5889.602, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:11:14+09:00", "last_error_summary": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 268, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:11:14+09:00", "last_error_summary": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "last_exit_at": "2026-07-06T18:11:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 204.005, "last_started_at": "2026-07-06T18:07:50+09:00", "pid": 21158, "process": {"exists": false, "pid": 21158}, "restart_count": 66, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.639, "sample_count": 1889, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 157261, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T18:11:14+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-06T18:11:14+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T18:11:14.848259+09:00", "mtime_epoch": 1783329074.8482587, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:11:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1613.312, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.834, "pushed": 2.834, "read": 2.801}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.168, "count": 84, "max_ms": 8.608, "p95_ms": 7.481}, "gstreamer_push": {"avg_ms": 1.093, "count": 85, "max_ms": 1.296, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.598, "count": 84, "max_ms": 78.997, "p95_ms": 73.099}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.522, "count": 84, "max_ms": 100.893, "p95_ms": 95.222}, "yolo11_640_fire_11class": {"avg_ms": 60.457, "count": 84, "max_ms": 86.587, "p95_ms": 80.05}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.355, "count": 84, "max_ms": 95.84, "p95_ms": 90.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.505, "count": 85, "max_ms": 155.28, "p95_ms": 150.524}, "read_interval": {"avg_ms": 357.104, "count": 84, "max_ms": 10122.713, "p95_ms": 163.753}, "triton_wait": {"avg_ms": 72.264, "count": 84, "max_ms": 100.906, "p95_ms": 95.79}, "visualize": {"avg_ms": 5.202, "count": 84, "max_ms": 6.489, "p95_ms": 5.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1310}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5889.684, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 268, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:11:14+09:00", "last_error_summary": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "last_exit_at": "2026-07-06T18:11:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 204.01, "last_started_at": "2026-07-06T18:07:50+09:00", "pid": 21158, "process": {"exists": false, "pid": 21158}, "restart_count": 66, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.639, "sample_count": 1889, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 157261, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T18:11:14+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-06T18:11:14+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-06T18:11:14.848259+09:00", "mtime_epoch": 1783329074.8482587, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:11:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1613.312, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.834, "pushed": 2.834, "read": 2.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.168, "count": 84, "max_ms": 8.608, "p95_ms": 7.481}, "gstreamer_push": {"avg_ms": 1.093, "count": 85, "max_ms": 1.296, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.598, "count": 84, "max_ms": 78.997, "p95_ms": 73.099}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.522, "count": 84, "max_ms": 100.893, "p95_ms": 95.222}, "yolo11_640_fire_11class": {"avg_ms": 60.457, "count": 84, "max_ms": 86.587, "p95_ms": 80.05}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.355, "count": 84, "max_ms": 95.84, "p95_ms": 90.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.505, "count": 85, "max_ms": 155.28, "p95_ms": 150.524}, "read_interval": {"avg_ms": 357.104, "count": 84, "max_ms": 10122.713, "p95_ms": 163.753}, "triton_wait": {"avg_ms": 72.264, "count": 84, "max_ms": 100.906, "p95_ms": 95.79}, "visualize": {"avg_ms": 5.202, "count": 84, "max_ms": 6.489, "p95_ms": 5.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1310}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6002.875, "limit_mb": 8796093022207.996}, "detail": {"pid": 21265, "restart_count": 67, "start_count": 68}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 268, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:11:14+09:00", "last_error_summary": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "last_exit_at": "2026-07-06T18:11:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:11:17+09:00", "pid": 21265, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.301, "name": "ffmpeg", "pid": 21265, "rss_mb": 141.301, "state": "S (sleeping)", "threads": 106}, "restart_count": 67, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.639, "sample_count": 1860, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 157261, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.25, "last_frame_pushed_at": "2026-07-06T18:11:14+09:00", "last_frame_read_age_sec": 4.373, "last_frame_read_at": "2026-07-06T18:11:14+09:00", "mjpeg": {"age_sec": 4.235, "exists": true, "mtime": "2026-07-06T18:11:14.980262+09:00", "mtime_epoch": 1783329074.9802623, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179573}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:11:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1613.312, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.774, "pushed": 1.774, "read": 1.74}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.155, "count": 53, "max_ms": 8.608, "p95_ms": 7.465}, "gstreamer_push": {"avg_ms": 1.083, "count": 53, "max_ms": 1.296, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.976, "count": 53, "max_ms": 76.666, "p95_ms": 71.656}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.972, "count": 53, "max_ms": 100.893, "p95_ms": 96.397}, "yolo11_640_fire_11class": {"avg_ms": 61.054, "count": 53, "max_ms": 86.587, "p95_ms": 82.877}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.85, "count": 53, "max_ms": 95.84, "p95_ms": 90.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.098, "count": 53, "max_ms": 155.28, "p95_ms": 151.429}, "read_interval": {"avg_ms": 493.202, "count": 52, "max_ms": 10122.713, "p95_ms": 164.324}, "triton_wait": {"avg_ms": 71.982, "count": 53, "max_ms": 100.906, "p95_ms": 96.409}, "visualize": {"avg_ms": 5.204, "count": 53, "max_ms": 6.489, "p95_ms": 5.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1321}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5894.746, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:11:14+09:00", "last_error_summary": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 268, "h264_decode_error_count_per_hour": 61, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:11:14+09:00", "last_error_summary": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "last_exit_at": "2026-07-06T18:14:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.848, "last_started_at": "2026-07-06T18:11:17+09:00", "pid": 21265, "process": {"exists": false, "pid": 21265}, "restart_count": 67, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.625, "sample_count": 2003, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 158573, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:14:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T18:14:23+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T18:14:22+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T18:14:22.945136+09:00", "mtime_epoch": 1783329262.9451356, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184392}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:14:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1613.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.201, "pushed": 7.201, "read": 7.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.152, "count": 216, "max_ms": 9.275, "p95_ms": 7.474}, "gstreamer_push": {"avg_ms": 1.102, "count": 216, "max_ms": 1.431, "p95_ms": 1.227}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.886, "count": 216, "max_ms": 86.775, "p95_ms": 76.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.031, "count": 216, "max_ms": 118.93, "p95_ms": 99.547}, "yolo11_640_fire_11class": {"avg_ms": 61.97, "count": 216, "max_ms": 92.292, "p95_ms": 83.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.441, "count": 216, "max_ms": 104.449, "p95_ms": 93.004}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.527, "count": 216, "max_ms": 171.222, "p95_ms": 155.406}, "read_interval": {"avg_ms": 138.855, "count": 216, "max_ms": 183.092, "p95_ms": 164.663}, "triton_wait": {"avg_ms": 76.225, "count": 216, "max_ms": 118.943, "p95_ms": 99.56}, "visualize": {"avg_ms": 5.159, "count": 216, "max_ms": 8.051, "p95_ms": 5.784}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1321}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5894.738, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 268, "h264_decode_error_count_per_hour": 61, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:11:14+09:00", "last_error_summary": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "last_exit_at": "2026-07-06T18:14:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.854, "last_started_at": "2026-07-06T18:11:17+09:00", "pid": 21265, "process": {"exists": false, "pid": 21265}, "restart_count": 67, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.625, "sample_count": 2003, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 158573, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:14:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T18:14:23+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T18:14:22+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T18:14:22.945136+09:00", "mtime_epoch": 1783329262.9451356, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184392}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:14:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1613.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.23, "pushed": 7.23, "read": 7.196}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.152, "count": 216, "max_ms": 9.275, "p95_ms": 7.474}, "gstreamer_push": {"avg_ms": 1.102, "count": 216, "max_ms": 1.431, "p95_ms": 1.227}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.886, "count": 216, "max_ms": 86.775, "p95_ms": 76.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.031, "count": 216, "max_ms": 118.93, "p95_ms": 99.547}, "yolo11_640_fire_11class": {"avg_ms": 61.97, "count": 216, "max_ms": 92.292, "p95_ms": 83.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.441, "count": 216, "max_ms": 104.449, "p95_ms": 93.004}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.527, "count": 216, "max_ms": 171.222, "p95_ms": 155.406}, "read_interval": {"avg_ms": 138.92, "count": 215, "max_ms": 183.092, "p95_ms": 164.663}, "triton_wait": {"avg_ms": 76.225, "count": 216, "max_ms": 118.943, "p95_ms": 99.56}, "visualize": {"avg_ms": 5.159, "count": 216, "max_ms": 8.051, "p95_ms": 5.784}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1321}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5995.48, "limit_mb": 8796093022207.996}, "detail": {"pid": 21372, "restart_count": 68, "start_count": 69}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 268, "h264_decode_error_count_per_hour": 61, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:11:14+09:00", "last_error_summary": "[h264 @ 0x557823262c80] left block unavailable for requested intra mode [h264 @ 0x557823262c80] error while decoding MB 0 24, bytestream 1018 [h264 @ 0x557823262c80] reference picture missing during reorder [h264 @ 0x557823262c80] Missing reference picture, default is 65552 [h264 @ 0x557823331b00] mmco: unref short failure [h264 @ 0x557823331b00] illegal short term buffer state detected [h264 @ 0x557823193ec0] left block unavailable for requested intra mode [h264 @ 0x557823193ec0] error while de", "last_exit_at": "2026-07-06T18:14:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:14:25+09:00", "pid": 21372, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.383, "name": "ffmpeg", "pid": 21372, "rss_mb": 136.383, "state": "S (sleeping)", "threads": 106}, "restart_count": 68, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.625, "sample_count": 1972, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 158573, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:14:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.278, "last_frame_pushed_at": "2026-07-06T18:14:23+09:00", "last_frame_read_age_sec": 4.401, "last_frame_read_at": "2026-07-06T18:14:22+09:00", "mjpeg": {"age_sec": 4.262, "exists": true, "mtime": "2026-07-06T18:14:23.073139+09:00", "mtime_epoch": 1783329263.0731387, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184576}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:14:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1613.828, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.189, "pushed": 6.189, "read": 6.156}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.152, "count": 185, "max_ms": 9.275, "p95_ms": 7.328}, "gstreamer_push": {"avg_ms": 1.106, "count": 185, "max_ms": 1.431, "p95_ms": 1.227}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.882, "count": 185, "max_ms": 86.775, "p95_ms": 76.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.172, "count": 185, "max_ms": 116.37, "p95_ms": 99.547}, "yolo11_640_fire_11class": {"avg_ms": 61.875, "count": 185, "max_ms": 89.988, "p95_ms": 83.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.204, "count": 185, "max_ms": 104.449, "p95_ms": 93.004}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.71, "count": 185, "max_ms": 171.057, "p95_ms": 154.227}, "read_interval": {"avg_ms": 139.158, "count": 184, "max_ms": 182.232, "p95_ms": 163.076}, "triton_wait": {"avg_ms": 76.363, "count": 185, "max_ms": 116.383, "p95_ms": 99.56}, "visualize": {"avg_ms": 5.169, "count": 185, "max_ms": 8.051, "p95_ms": 6.108}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1331}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5897.355, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55eaf7ab8ac0] error while decoding MB 2 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 269, "h264_decode_error_count_per_hour": 59, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:17:24+09:00", "last_error_summary": "[h264 @ 0x55eaf7ab8ac0] error while decoding MB 2 37, bytestream -5", "last_exit_at": "2026-07-06T18:14:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.704, "last_started_at": "2026-07-06T18:14:25+09:00", "pid": 21372, "process": {"exists": false, "pid": 21372}, "restart_count": 68, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.723, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 159817, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:17:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T18:17:23+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T18:17:23+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T18:17:23.993416+09:00", "mtime_epoch": 1783329443.9934158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181503}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:17:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1614.602, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.192, "pushed": 7.192, "read": 7.192}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.295, "count": 215, "max_ms": 9.088, "p95_ms": 7.656}, "gstreamer_push": {"avg_ms": 1.096, "count": 215, "max_ms": 1.296, "p95_ms": 1.223}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.24, "count": 215, "max_ms": 82.363, "p95_ms": 75.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.396, "count": 215, "max_ms": 110.027, "p95_ms": 99.396}, "yolo11_640_fire_11class": {"avg_ms": 61.223, "count": 215, "max_ms": 93.46, "p95_ms": 80.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.632, "count": 215, "max_ms": 104.247, "p95_ms": 92.106}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.866, "count": 215, "max_ms": 191.433, "p95_ms": 157.548}, "read_interval": {"avg_ms": 139.344, "count": 215, "max_ms": 199.508, "p95_ms": 167.507}, "triton_wait": {"avg_ms": 74.757, "count": 215, "max_ms": 110.039, "p95_ms": 100.365}, "visualize": {"avg_ms": 5.257, "count": 215, "max_ms": 7.356, "p95_ms": 6.322}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1331}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5896.59, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:17:24+09:00", "last_error_summary": "[h264 @ 0x55eaf7ab8ac0] error while decoding MB 2 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 269, "h264_decode_error_count_per_hour": 59, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:17:24+09:00", "last_error_summary": "[h264 @ 0x55eaf7ab8ac0] error while decoding MB 2 37, bytestream -5", "last_exit_at": "2026-07-06T18:17:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.791, "last_started_at": "2026-07-06T18:14:25+09:00", "pid": 21372, "process": {"exists": false, "pid": 21372}, "restart_count": 68, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.723, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 159818, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:17:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:17:24+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T18:17:23+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-06T18:17:23.993416+09:00", "mtime_epoch": 1783329443.9934158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181503}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:17:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1614.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.205, "pushed": 7.205, "read": 7.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.296, "count": 215, "max_ms": 9.088, "p95_ms": 7.656}, "gstreamer_push": {"avg_ms": 1.096, "count": 216, "max_ms": 1.296, "p95_ms": 1.223}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.307, "count": 215, "max_ms": 82.363, "p95_ms": 75.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.455, "count": 215, "max_ms": 110.027, "p95_ms": 99.396}, "yolo11_640_fire_11class": {"avg_ms": 61.26, "count": 215, "max_ms": 93.46, "p95_ms": 80.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.702, "count": 215, "max_ms": 104.247, "p95_ms": 92.106}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.879, "count": 216, "max_ms": 191.433, "p95_ms": 157.548}, "read_interval": {"avg_ms": 139.344, "count": 215, "max_ms": 199.508, "p95_ms": 167.507}, "triton_wait": {"avg_ms": 74.816, "count": 215, "max_ms": 110.039, "p95_ms": 100.365}, "visualize": {"avg_ms": 5.257, "count": 215, "max_ms": 7.356, "p95_ms": 6.322}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1331}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5896.484, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 269, "h264_decode_error_count_per_hour": 59, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:17:24+09:00", "last_error_summary": "[h264 @ 0x55eaf7ab8ac0] error while decoding MB 2 37, bytestream -5", "last_exit_at": "2026-07-06T18:17:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 178.797, "last_started_at": "2026-07-06T18:14:25+09:00", "pid": 21372, "process": {"exists": false, "pid": 21372}, "restart_count": 68, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.723, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 159818, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:17:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T18:17:24+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-06T18:17:23+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-06T18:17:23.993416+09:00", "mtime_epoch": 1783329443.9934158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181503}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:17:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1614.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.203, "pushed": 7.203, "read": 7.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.296, "count": 215, "max_ms": 9.088, "p95_ms": 7.656}, "gstreamer_push": {"avg_ms": 1.096, "count": 216, "max_ms": 1.296, "p95_ms": 1.223}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.307, "count": 215, "max_ms": 82.363, "p95_ms": 75.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.455, "count": 215, "max_ms": 110.027, "p95_ms": 99.396}, "yolo11_640_fire_11class": {"avg_ms": 61.26, "count": 215, "max_ms": 93.46, "p95_ms": 80.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.702, "count": 215, "max_ms": 104.247, "p95_ms": 92.106}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.879, "count": 216, "max_ms": 191.433, "p95_ms": 157.548}, "read_interval": {"avg_ms": 139.344, "count": 215, "max_ms": 199.508, "p95_ms": 167.507}, "triton_wait": {"avg_ms": 74.816, "count": 215, "max_ms": 110.039, "p95_ms": 100.365}, "visualize": {"avg_ms": 5.257, "count": 215, "max_ms": 7.356, "p95_ms": 6.322}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1331}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6013.234, "limit_mb": 8796093022207.996}, "detail": {"pid": 21479, "restart_count": 69, "start_count": 70}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 269, "h264_decode_error_count_per_hour": 59, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:17:24+09:00", "last_error_summary": "[h264 @ 0x55eaf7ab8ac0] error while decoding MB 2 37, bytestream -5", "last_exit_at": "2026-07-06T18:17:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:17:27+09:00", "pid": 21479, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.07, "name": "ffmpeg", "pid": 21479, "rss_mb": 147.07, "state": "S (sleeping)", "threads": 106}, "restart_count": 69, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.059, "sample_count": 2056, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 159818, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:17:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.903, "last_frame_pushed_at": "2026-07-06T18:17:24+09:00", "last_frame_read_age_sec": 5.037, "last_frame_read_at": "2026-07-06T18:17:23+09:00", "mjpeg": {"age_sec": 4.885, "exists": true, "mtime": "2026-07-06T18:17:24.137419+09:00", "mtime_epoch": 1783329444.1374192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179636}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:17:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1614.602, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.036, "pushed": 6.036, "read": 6.002}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.332, "count": 180, "max_ms": 9.088, "p95_ms": 7.656}, "gstreamer_push": {"avg_ms": 1.097, "count": 181, "max_ms": 1.296, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.252, "count": 180, "max_ms": 82.363, "p95_ms": 75.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.589, "count": 180, "max_ms": 110.027, "p95_ms": 100.353}, "yolo11_640_fire_11class": {"avg_ms": 61.385, "count": 180, "max_ms": 93.46, "p95_ms": 81.15}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.918, "count": 180, "max_ms": 104.247, "p95_ms": 91.663}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.852, "count": 181, "max_ms": 191.433, "p95_ms": 157.548}, "read_interval": {"avg_ms": 139.382, "count": 180, "max_ms": 199.508, "p95_ms": 167.507}, "triton_wait": {"avg_ms": 75.006, "count": 180, "max_ms": 110.039, "p95_ms": 102.287}, "visualize": {"avg_ms": 5.254, "count": 180, "max_ms": 7.141, "p95_ms": 6.233}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1342}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5902.875, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 59, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:17:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.827, "last_started_at": "2026-07-06T18:17:27+09:00", "pid": 21479, "process": {"exists": false, "pid": 21479}, "restart_count": 69, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.876, "sample_count": 2097, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 161135, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:20:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T18:20:32+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T18:20:32+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T18:20:32.801579+09:00", "mtime_epoch": 1783329632.8015795, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:20:32+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1615.117, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.149, "pushed": 7.149, "read": 7.149}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.156, "count": 214, "max_ms": 9.587, "p95_ms": 7.322}, "gstreamer_push": {"avg_ms": 1.098, "count": 214, "max_ms": 1.318, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.2, "count": 214, "max_ms": 86.354, "p95_ms": 78.154}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.911, "count": 214, "max_ms": 122.677, "p95_ms": 104.246}, "yolo11_640_fire_11class": {"avg_ms": 63.075, "count": 214, "max_ms": 93.777, "p95_ms": 88.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.531, "count": 214, "max_ms": 102.811, "p95_ms": 94.918}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.551, "count": 214, "max_ms": 189.616, "p95_ms": 161.786}, "read_interval": {"avg_ms": 140.12, "count": 214, "max_ms": 198.028, "p95_ms": 172.422}, "triton_wait": {"avg_ms": 76.26, "count": 214, "max_ms": 122.69, "p95_ms": 104.259}, "visualize": {"avg_ms": 5.177, "count": 214, "max_ms": 8.511, "p95_ms": 6.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1342}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5902.188, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 59, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:20:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.9, "last_started_at": "2026-07-06T18:17:27+09:00", "pid": 21479, "process": {"exists": false, "pid": 21479}, "restart_count": 69, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.876, "sample_count": 2097, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 161136, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:20:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T18:20:32+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T18:20:32+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T18:20:32.801579+09:00", "mtime_epoch": 1783329632.8015795, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:20:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1615.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.134, "pushed": 7.134, "read": 7.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.182, "count": 214, "max_ms": 9.592, "p95_ms": 7.436}, "gstreamer_push": {"avg_ms": 1.099, "count": 214, "max_ms": 1.318, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.159, "count": 214, "max_ms": 86.354, "p95_ms": 78.154}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.87, "count": 214, "max_ms": 122.677, "p95_ms": 104.246}, "yolo11_640_fire_11class": {"avg_ms": 63.049, "count": 214, "max_ms": 93.777, "p95_ms": 88.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.477, "count": 214, "max_ms": 102.811, "p95_ms": 94.918}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.61, "count": 214, "max_ms": 189.616, "p95_ms": 161.786}, "read_interval": {"avg_ms": 140.12, "count": 214, "max_ms": 198.028, "p95_ms": 172.422}, "triton_wait": {"avg_ms": 76.216, "count": 214, "max_ms": 122.69, "p95_ms": 104.259}, "visualize": {"avg_ms": 5.187, "count": 214, "max_ms": 8.511, "p95_ms": 6.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1342}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5902.18, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 59, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:20:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.907, "last_started_at": "2026-07-06T18:17:27+09:00", "pid": 21479, "process": {"exists": false, "pid": 21479}, "restart_count": 69, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.876, "sample_count": 2097, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 161136, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:20:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T18:20:32+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T18:20:32+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-06T18:20:32.801579+09:00", "mtime_epoch": 1783329632.8015795, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:20:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1615.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.16, "pushed": 7.16, "read": 7.126}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.182, "count": 214, "max_ms": 9.592, "p95_ms": 7.436}, "gstreamer_push": {"avg_ms": 1.099, "count": 214, "max_ms": 1.318, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.159, "count": 214, "max_ms": 86.354, "p95_ms": 78.154}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.87, "count": 214, "max_ms": 122.677, "p95_ms": 104.246}, "yolo11_640_fire_11class": {"avg_ms": 63.049, "count": 214, "max_ms": 93.777, "p95_ms": 88.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.477, "count": 214, "max_ms": 102.811, "p95_ms": 94.918}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.61, "count": 214, "max_ms": 189.616, "p95_ms": 161.786}, "read_interval": {"avg_ms": 140.232, "count": 213, "max_ms": 198.028, "p95_ms": 172.422}, "triton_wait": {"avg_ms": 76.216, "count": 214, "max_ms": 122.69, "p95_ms": 104.259}, "visualize": {"avg_ms": 5.187, "count": 214, "max_ms": 8.511, "p95_ms": 6.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1342}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6009.559, "limit_mb": 8796093022207.996}, "detail": {"pid": 21586, "restart_count": 70, "start_count": 71}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 59, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:20:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:20:35+09:00", "pid": 21586, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.281, "name": "ffmpeg", "pid": 21586, "rss_mb": 139.281, "state": "S (sleeping)", "threads": 106}, "restart_count": 70, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.324, "sample_count": 2065, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 161136, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:20:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.511, "last_frame_pushed_at": "2026-07-06T18:20:32+09:00", "last_frame_read_age_sec": 4.634, "last_frame_read_at": "2026-07-06T18:20:32+09:00", "mjpeg": {"age_sec": 4.493, "exists": true, "mtime": "2026-07-06T18:20:32.933582+09:00", "mtime_epoch": 1783329632.9335823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:20:37+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1615.117, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 182, "pushed_frames": 182, "read_frames": 181, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.074, "pushed": 6.074, "read": 6.041}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.239, "count": 182, "max_ms": 9.592, "p95_ms": 7.322}, "gstreamer_push": {"avg_ms": 1.104, "count": 182, "max_ms": 1.318, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.812, "count": 182, "max_ms": 86.354, "p95_ms": 76.701}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.718, "count": 182, "max_ms": 122.677, "p95_ms": 104.266}, "yolo11_640_fire_11class": {"avg_ms": 62.802, "count": 182, "max_ms": 93.777, "p95_ms": 87.687}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.118, "count": 182, "max_ms": 102.811, "p95_ms": 94.866}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.894, "count": 182, "max_ms": 189.616, "p95_ms": 164.737}, "read_interval": {"avg_ms": 140.535, "count": 181, "max_ms": 198.028, "p95_ms": 174.186}, "triton_wait": {"avg_ms": 76.077, "count": 182, "max_ms": 122.69, "p95_ms": 104.28}, "visualize": {"avg_ms": 5.244, "count": 182, "max_ms": 8.511, "p95_ms": 6.284}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1353}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5898.988, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 58, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:23:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.583, "last_started_at": "2026-07-06T18:20:35+09:00", "pid": 21586, "process": {"exists": false, "pid": 21586}, "restart_count": 70, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.967, "sample_count": 2116, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 162441, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:23:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T18:23:40+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-06T18:23:39+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T18:23:39.885494+09:00", "mtime_epoch": 1783329819.8854945, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186485}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:23:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1615.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.138, "pushed": 7.138, "read": 7.104}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.308, "count": 213, "max_ms": 9.658, "p95_ms": 7.268}, "gstreamer_push": {"avg_ms": 1.086, "count": 214, "max_ms": 1.668, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.095, "count": 213, "max_ms": 80.3, "p95_ms": 74.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.794, "count": 213, "max_ms": 113.381, "p95_ms": 100.405}, "yolo11_640_fire_11class": {"avg_ms": 63.108, "count": 213, "max_ms": 93.164, "p95_ms": 86.691}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.815, "count": 213, "max_ms": 102.686, "p95_ms": 94.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.281, "count": 214, "max_ms": 187.214, "p95_ms": 164.534}, "read_interval": {"avg_ms": 140.715, "count": 213, "max_ms": 194.76, "p95_ms": 173.146}, "triton_wait": {"avg_ms": 75.023, "count": 213, "max_ms": 113.394, "p95_ms": 100.417}, "visualize": {"avg_ms": 5.368, "count": 213, "max_ms": 8.596, "p95_ms": 6.404}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1353}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5901.582, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 58, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:23:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.591, "last_started_at": "2026-07-06T18:20:35+09:00", "pid": 21586, "process": {"exists": false, "pid": 21586}, "restart_count": 70, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.967, "sample_count": 2116, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 162441, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:23:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T18:23:40+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-06T18:23:39+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-06T18:23:39.885494+09:00", "mtime_epoch": 1783329819.8854945, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186485}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:23:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1615.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.136, "pushed": 7.136, "read": 7.102}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.308, "count": 213, "max_ms": 9.658, "p95_ms": 7.268}, "gstreamer_push": {"avg_ms": 1.086, "count": 214, "max_ms": 1.668, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.095, "count": 213, "max_ms": 80.3, "p95_ms": 74.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.794, "count": 213, "max_ms": 113.381, "p95_ms": 100.405}, "yolo11_640_fire_11class": {"avg_ms": 63.108, "count": 213, "max_ms": 93.164, "p95_ms": 86.691}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.815, "count": 213, "max_ms": 102.686, "p95_ms": 94.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.281, "count": 214, "max_ms": 187.214, "p95_ms": 164.534}, "read_interval": {"avg_ms": 140.715, "count": 213, "max_ms": 194.76, "p95_ms": 173.146}, "triton_wait": {"avg_ms": 75.023, "count": 213, "max_ms": 113.394, "p95_ms": 100.417}, "visualize": {"avg_ms": 5.368, "count": 213, "max_ms": 8.596, "p95_ms": 6.404}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1353}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6009.574, "limit_mb": 8796093022207.996}, "detail": {"pid": 21693, "restart_count": 71, "start_count": 72}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 58, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:23:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:23:42+09:00", "pid": 21693, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.34, "name": "ffmpeg", "pid": 21693, "rss_mb": 139.34, "state": "S (sleeping)", "threads": 106}, "restart_count": 71, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 157.703, "sample_count": 2087, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 162441, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:23:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.178, "last_frame_pushed_at": "2026-07-06T18:23:40+09:00", "last_frame_read_age_sec": 4.304, "last_frame_read_at": "2026-07-06T18:23:39+09:00", "mjpeg": {"age_sec": 4.159, "exists": true, "mtime": "2026-07-06T18:23:40.021497+09:00", "mtime_epoch": 1783329820.0214972, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186772}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:23:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1615.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.174, "pushed": 6.174, "read": 6.14}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.304, "count": 184, "max_ms": 9.658, "p95_ms": 7.268}, "gstreamer_push": {"avg_ms": 1.088, "count": 185, "max_ms": 1.668, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.275, "count": 184, "max_ms": 80.3, "p95_ms": 74.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.749, "count": 184, "max_ms": 113.381, "p95_ms": 100.405}, "yolo11_640_fire_11class": {"avg_ms": 62.906, "count": 184, "max_ms": 93.164, "p95_ms": 87.487}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.708, "count": 184, "max_ms": 102.686, "p95_ms": 94.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.806, "count": 185, "max_ms": 187.214, "p95_ms": 163.75}, "read_interval": {"avg_ms": 140.246, "count": 184, "max_ms": 194.76, "p95_ms": 173.146}, "triton_wait": {"avg_ms": 75.012, "count": 184, "max_ms": 113.394, "p95_ms": 100.417}, "visualize": {"avg_ms": 5.342, "count": 184, "max_ms": 8.596, "p95_ms": 6.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5903.602, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 56, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:26:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 188.16, "last_started_at": "2026-07-06T18:23:42+09:00", "pid": 21693, "process": {"exists": false, "pid": 21693}, "restart_count": 71, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.064, "sample_count": 2130, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 163784, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:26:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:26:50+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-06T18:26:50+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-06T18:26:50.229329+09:00", "mtime_epoch": 1783330010.2293289, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:26:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1616.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.29, "pushed": 7.29, "read": 7.257}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.222, "count": 218, "max_ms": 9.462, "p95_ms": 7.713}, "gstreamer_push": {"avg_ms": 1.12, "count": 218, "max_ms": 1.73, "p95_ms": 1.241}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.328, "count": 218, "max_ms": 82.644, "p95_ms": 74.449}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.251, "count": 218, "max_ms": 115.368, "p95_ms": 99.171}, "yolo11_640_fire_11class": {"avg_ms": 61.667, "count": 218, "max_ms": 96.992, "p95_ms": 87.475}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.308, "count": 218, "max_ms": 101.129, "p95_ms": 92.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.302, "count": 218, "max_ms": 174.988, "p95_ms": 154.582}, "read_interval": {"avg_ms": 137.947, "count": 217, "max_ms": 183.147, "p95_ms": 163.898}, "triton_wait": {"avg_ms": 74.586, "count": 218, "max_ms": 115.384, "p95_ms": 99.58}, "visualize": {"avg_ms": 5.222, "count": 218, "max_ms": 8.064, "p95_ms": 6.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5903.719, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 56, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:26:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 188.166, "last_started_at": "2026-07-06T18:23:42+09:00", "pid": 21693, "process": {"exists": false, "pid": 21693}, "restart_count": 71, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.064, "sample_count": 2130, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 163784, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:26:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T18:26:50+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T18:26:50+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T18:26:50.229329+09:00", "mtime_epoch": 1783330010.2293289, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:26:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1616.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.289, "pushed": 7.289, "read": 7.255}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.222, "count": 218, "max_ms": 9.462, "p95_ms": 7.713}, "gstreamer_push": {"avg_ms": 1.12, "count": 218, "max_ms": 1.73, "p95_ms": 1.241}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.328, "count": 218, "max_ms": 82.644, "p95_ms": 74.449}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.251, "count": 218, "max_ms": 115.368, "p95_ms": 99.171}, "yolo11_640_fire_11class": {"avg_ms": 61.667, "count": 218, "max_ms": 96.992, "p95_ms": 87.475}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.308, "count": 218, "max_ms": 101.129, "p95_ms": 92.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.302, "count": 218, "max_ms": 174.988, "p95_ms": 154.582}, "read_interval": {"avg_ms": 137.947, "count": 217, "max_ms": 183.147, "p95_ms": 163.898}, "triton_wait": {"avg_ms": 74.586, "count": 218, "max_ms": 115.384, "p95_ms": 99.58}, "visualize": {"avg_ms": 5.222, "count": 218, "max_ms": 8.064, "p95_ms": 6.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6015.395, "limit_mb": 8796093022207.996}, "detail": {"pid": 21800, "restart_count": 72, "start_count": 73}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 270, "h264_decode_error_count_per_hour": 56, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:20:32+09:00", "last_error_summary": "[h264 @ 0x55d1627255c0] error while decoding MB 28 20, bytestream -5", "last_exit_at": "2026-07-06T18:26:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:26:52+09:00", "pid": 21800, "process": {"exists": true, "fd_count": 5, "hwm_mb": 143.98, "name": "ffmpeg", "pid": 21800, "rss_mb": 143.98, "state": "S (sleeping)", "threads": 106}, "restart_count": 72, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.195, "sample_count": 2098, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 163784, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:26:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.565, "last_frame_pushed_at": "2026-07-06T18:26:50+09:00", "last_frame_read_age_sec": 4.68, "last_frame_read_at": "2026-07-06T18:26:50+09:00", "mjpeg": {"age_sec": 4.55, "exists": true, "mtime": "2026-07-06T18:26:50.349331+09:00", "mtime_epoch": 1783330010.3493314, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180814}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:26:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1616.406, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 186, "pushed_frames": 186, "read_frames": 185, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.213, "pushed": 6.213, "read": 6.18}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.257, "count": 186, "max_ms": 9.462, "p95_ms": 7.713}, "gstreamer_push": {"avg_ms": 1.122, "count": 186, "max_ms": 1.73, "p95_ms": 1.244}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.947, "count": 186, "max_ms": 82.644, "p95_ms": 73.948}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.509, "count": 186, "max_ms": 115.368, "p95_ms": 97.654}, "yolo11_640_fire_11class": {"avg_ms": 61.197, "count": 186, "max_ms": 96.992, "p95_ms": 87.083}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.909, "count": 186, "max_ms": 99.826, "p95_ms": 91.81}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.602, "count": 186, "max_ms": 174.988, "p95_ms": 153.306}, "read_interval": {"avg_ms": 137.225, "count": 185, "max_ms": 183.147, "p95_ms": 162.299}, "triton_wait": {"avg_ms": 73.882, "count": 186, "max_ms": 115.384, "p95_ms": 98.297}, "visualize": {"avg_ms": 5.198, "count": 186, "max_ms": 7.338, "p95_ms": 6.155}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1376}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5906.641, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560433736300] error while decoding MB 6 2, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 271, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:30:02+09:00", "last_error_summary": "[h264 @ 0x560433736300] error while decoding MB 6 2, bytestream -6", "last_exit_at": "2026-07-06T18:26:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.563, "last_started_at": "2026-07-06T18:26:52+09:00", "pid": 21800, "process": {"exists": false, "pid": 21800}, "restart_count": 72, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.756, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 165142, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:30:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T18:30:02+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T18:30:02+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T18:30:02.417095+09:00", "mtime_epoch": 1783330202.4170954, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184141}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:30:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1616.922, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 219, "pushed_frames": 219, "read_frames": 219, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.327, "pushed": 7.327, "read": 7.327}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.052, "count": 219, "max_ms": 9.366, "p95_ms": 7.48}, "gstreamer_push": {"avg_ms": 1.097, "count": 219, "max_ms": 1.744, "p95_ms": 1.242}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.582, "count": 219, "max_ms": 80.934, "p95_ms": 75.575}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.605, "count": 219, "max_ms": 112.074, "p95_ms": 98.932}, "yolo11_640_fire_11class": {"avg_ms": 62.333, "count": 219, "max_ms": 93.781, "p95_ms": 85.223}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.599, "count": 219, "max_ms": 101.58, "p95_ms": 92.4}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.209, "count": 219, "max_ms": 168.492, "p95_ms": 154.764}, "read_interval": {"avg_ms": 136.77, "count": 219, "max_ms": 179.182, "p95_ms": 164.561}, "triton_wait": {"avg_ms": 74.87, "count": 219, "max_ms": 112.087, "p95_ms": 98.943}, "visualize": {"avg_ms": 5.062, "count": 219, "max_ms": 7.778, "p95_ms": 5.825}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1376}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5906.086, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:30:02+09:00", "last_error_summary": "[h264 @ 0x560433736300] error while decoding MB 6 2, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 271, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:30:02+09:00", "last_error_summary": "[h264 @ 0x560433736300] error while decoding MB 6 2, bytestream -6", "last_exit_at": "2026-07-06T18:30:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.648, "last_started_at": "2026-07-06T18:26:52+09:00", "pid": 21800, "process": {"exists": false, "pid": 21800}, "restart_count": 72, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.756, "sample_count": 2139, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 165143, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:30:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:30:02+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-06T18:30:02+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T18:30:02.417095+09:00", "mtime_epoch": 1783330202.4170954, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184141}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:30:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1616.922, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 219, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.34, "pushed": 7.34, "read": 7.306}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.052, "count": 219, "max_ms": 9.366, "p95_ms": 7.48}, "gstreamer_push": {"avg_ms": 1.096, "count": 220, "max_ms": 1.744, "p95_ms": 1.238}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.673, "count": 219, "max_ms": 80.934, "p95_ms": 75.575}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.668, "count": 219, "max_ms": 112.074, "p95_ms": 98.932}, "yolo11_640_fire_11class": {"avg_ms": 62.378, "count": 219, "max_ms": 93.781, "p95_ms": 85.223}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.668, "count": 219, "max_ms": 101.58, "p95_ms": 92.4}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.223, "count": 220, "max_ms": 168.492, "p95_ms": 153.817}, "read_interval": {"avg_ms": 136.77, "count": 219, "max_ms": 179.182, "p95_ms": 164.561}, "triton_wait": {"avg_ms": 74.935, "count": 219, "max_ms": 112.087, "p95_ms": 98.943}, "visualize": {"avg_ms": 5.063, "count": 219, "max_ms": 7.778, "p95_ms": 5.825}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1376}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5906.133, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 271, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:30:02+09:00", "last_error_summary": "[h264 @ 0x560433736300] error while decoding MB 6 2, bytestream -6", "last_exit_at": "2026-07-06T18:30:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.655, "last_started_at": "2026-07-06T18:26:52+09:00", "pid": 21800, "process": {"exists": false, "pid": 21800}, "restart_count": 72, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.756, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 165143, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:30:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T18:30:02+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T18:30:02+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-06T18:30:02.417095+09:00", "mtime_epoch": 1783330202.4170954, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184141}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:30:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1616.922, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 219, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.338, "pushed": 7.338, "read": 7.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.052, "count": 219, "max_ms": 9.366, "p95_ms": 7.48}, "gstreamer_push": {"avg_ms": 1.096, "count": 220, "max_ms": 1.744, "p95_ms": 1.238}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.673, "count": 219, "max_ms": 80.934, "p95_ms": 75.575}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.668, "count": 219, "max_ms": 112.074, "p95_ms": 98.932}, "yolo11_640_fire_11class": {"avg_ms": 62.378, "count": 219, "max_ms": 93.781, "p95_ms": 85.223}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.668, "count": 219, "max_ms": 101.58, "p95_ms": 92.4}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.223, "count": 220, "max_ms": 168.492, "p95_ms": 153.817}, "read_interval": {"avg_ms": 136.77, "count": 219, "max_ms": 179.182, "p95_ms": 164.561}, "triton_wait": {"avg_ms": 74.935, "count": 219, "max_ms": 112.087, "p95_ms": 98.943}, "visualize": {"avg_ms": 5.063, "count": 219, "max_ms": 7.778, "p95_ms": 5.825}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1376}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6025.348, "limit_mb": 8796093022207.996}, "detail": {"pid": 21907, "restart_count": 73, "start_count": 74}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 271, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:30:02+09:00", "last_error_summary": "[h264 @ 0x560433736300] error while decoding MB 6 2, bytestream -6", "last_exit_at": "2026-07-06T18:30:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:30:05+09:00", "pid": 21907, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.363, "name": "ffmpeg", "pid": 21907, "rss_mb": 149.363, "state": "S (sleeping)", "threads": 106}, "restart_count": 73, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.757, "sample_count": 2106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 165143, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:30:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.53, "last_frame_pushed_at": "2026-07-06T18:30:02+09:00", "last_frame_read_age_sec": 4.662, "last_frame_read_at": "2026-07-06T18:30:02+09:00", "mjpeg": {"age_sec": 4.511, "exists": true, "mtime": "2026-07-06T18:30:02.561098+09:00", "mtime_epoch": 1783330202.5610983, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185022}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:30:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1616.922, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 186, "pushed_frames": 186, "read_frames": 185, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.226, "pushed": 6.226, "read": 6.193}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.987, "count": 186, "max_ms": 9.366, "p95_ms": 7.297}, "gstreamer_push": {"avg_ms": 1.098, "count": 186, "max_ms": 1.744, "p95_ms": 1.238}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.89, "count": 186, "max_ms": 80.934, "p95_ms": 75.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.269, "count": 186, "max_ms": 112.074, "p95_ms": 101.474}, "yolo11_640_fire_11class": {"avg_ms": 62.698, "count": 186, "max_ms": 93.781, "p95_ms": 85.92}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.711, "count": 186, "max_ms": 101.58, "p95_ms": 92.365}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.496, "count": 186, "max_ms": 168.492, "p95_ms": 154.802}, "read_interval": {"avg_ms": 137.048, "count": 185, "max_ms": 179.182, "p95_ms": 165.298}, "triton_wait": {"avg_ms": 75.398, "count": 186, "max_ms": 112.087, "p95_ms": 101.489}, "visualize": {"avg_ms": 5.04, "count": 186, "max_ms": 7.778, "p95_ms": 5.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5909.562, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5611a57dff40] error while decoding MB 37 19, bytestream -32", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 272, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:33:12+09:00", "last_error_summary": "[h264 @ 0x5611a57dff40] error while decoding MB 37 19, bytestream -32", "last_exit_at": "2026-07-06T18:30:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.489, "last_started_at": "2026-07-06T18:30:05+09:00", "pid": 21907, "process": {"exists": false, "pid": 21907}, "restart_count": 73, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.508, "sample_count": 2139, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 166481, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:33:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.041, "last_frame_pushed_at": "2026-07-06T18:33:12+09:00", "last_frame_read_age_sec": 0.034, "last_frame_read_at": "2026-07-06T18:33:12+09:00", "mjpeg": {"age_sec": 0.023, "exists": true, "mtime": "2026-07-06T18:33:12.536750+09:00", "mtime_epoch": 1783330392.53675, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:33:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1617.695, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.174, "pushed": 7.174, "read": 7.174}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.232, "count": 214, "max_ms": 9.581, "p95_ms": 7.936}, "gstreamer_push": {"avg_ms": 1.109, "count": 215, "max_ms": 1.487, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.588, "count": 214, "max_ms": 84.339, "p95_ms": 75.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.295, "count": 214, "max_ms": 121.041, "p95_ms": 98.205}, "yolo11_640_fire_11class": {"avg_ms": 62.705, "count": 214, "max_ms": 91.037, "p95_ms": 84.039}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.717, "count": 214, "max_ms": 101.443, "p95_ms": 93.159}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.179, "count": 215, "max_ms": 174.303, "p95_ms": 152.103}, "read_interval": {"avg_ms": 139.762, "count": 215, "max_ms": 183.59, "p95_ms": 162.668}, "triton_wait": {"avg_ms": 76.609, "count": 214, "max_ms": 121.053, "p95_ms": 98.219}, "visualize": {"avg_ms": 5.22, "count": 214, "max_ms": 8.581, "p95_ms": 6.132}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5908.789, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:33:12+09:00", "last_error_summary": "[h264 @ 0x5611a57dff40] error while decoding MB 37 19, bytestream -32", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 272, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:33:12+09:00", "last_error_summary": "[h264 @ 0x5611a57dff40] error while decoding MB 37 19, bytestream -32", "last_exit_at": "2026-07-06T18:33:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.611, "last_started_at": "2026-07-06T18:30:05+09:00", "pid": 21907, "process": {"exists": false, "pid": 21907}, "restart_count": 73, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.508, "sample_count": 2139, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 166482, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:33:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:33:12+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-06T18:33:12+09:00", "mjpeg": {"age_sec": 0.145, "exists": true, "mtime": "2026-07-06T18:33:12.536750+09:00", "mtime_epoch": 1783330392.53675, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:33:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1617.695, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.177, "pushed": 7.177, "read": 7.143}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.232, "count": 215, "max_ms": 9.581, "p95_ms": 7.936}, "gstreamer_push": {"avg_ms": 1.11, "count": 215, "max_ms": 1.487, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.62, "count": 215, "max_ms": 84.339, "p95_ms": 75.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.384, "count": 215, "max_ms": 121.041, "p95_ms": 98.205}, "yolo11_640_fire_11class": {"avg_ms": 62.735, "count": 215, "max_ms": 91.037, "p95_ms": 84.039}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.744, "count": 215, "max_ms": 101.443, "p95_ms": 93.159}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.347, "count": 215, "max_ms": 174.303, "p95_ms": 152.103}, "read_interval": {"avg_ms": 139.843, "count": 214, "max_ms": 183.59, "p95_ms": 162.668}, "triton_wait": {"avg_ms": 76.697, "count": 215, "max_ms": 121.053, "p95_ms": 98.219}, "visualize": {"avg_ms": 5.22, "count": 215, "max_ms": 8.581, "p95_ms": 6.132}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5908.781, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 272, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:33:12+09:00", "last_error_summary": "[h264 @ 0x5611a57dff40] error while decoding MB 37 19, bytestream -32", "last_exit_at": "2026-07-06T18:33:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.617, "last_started_at": "2026-07-06T18:30:05+09:00", "pid": 21907, "process": {"exists": false, "pid": 21907}, "restart_count": 73, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.508, "sample_count": 2139, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 166482, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:33:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T18:33:12+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-06T18:33:12+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-06T18:33:12.536750+09:00", "mtime_epoch": 1783330392.53675, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:33:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1617.695, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.175, "pushed": 7.175, "read": 7.142}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.232, "count": 214, "max_ms": 9.581, "p95_ms": 7.936}, "gstreamer_push": {"avg_ms": 1.11, "count": 215, "max_ms": 1.487, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.646, "count": 214, "max_ms": 84.339, "p95_ms": 75.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.411, "count": 214, "max_ms": 121.041, "p95_ms": 98.205}, "yolo11_640_fire_11class": {"avg_ms": 62.75, "count": 214, "max_ms": 91.037, "p95_ms": 84.039}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.766, "count": 214, "max_ms": 101.443, "p95_ms": 93.159}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.347, "count": 215, "max_ms": 174.303, "p95_ms": 152.103}, "read_interval": {"avg_ms": 139.843, "count": 214, "max_ms": 183.59, "p95_ms": 162.668}, "triton_wait": {"avg_ms": 76.726, "count": 214, "max_ms": 121.053, "p95_ms": 98.219}, "visualize": {"avg_ms": 5.214, "count": 214, "max_ms": 8.581, "p95_ms": 6.035}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6018.926, "limit_mb": 8796093022207.996}, "detail": {"pid": 22014, "restart_count": 74, "start_count": 75}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 272, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:33:12+09:00", "last_error_summary": "[h264 @ 0x5611a57dff40] error while decoding MB 37 19, bytestream -32", "last_exit_at": "2026-07-06T18:33:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:33:15+09:00", "pid": 22014, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.062, "name": "ffmpeg", "pid": 22014, "rss_mb": 141.062, "state": "S (sleeping)", "threads": 106}, "restart_count": 74, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.508, "sample_count": 2106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 166482, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:33:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.512, "last_frame_pushed_at": "2026-07-06T18:33:12+09:00", "last_frame_read_age_sec": 4.663, "last_frame_read_at": "2026-07-06T18:33:12+09:00", "mjpeg": {"age_sec": 4.496, "exists": true, "mtime": "2026-07-06T18:33:12.692753+09:00", "mtime_epoch": 1783330392.692753, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193646}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:33:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1617.695, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.105, "pushed": 6.105, "read": 6.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.277, "count": 182, "max_ms": 9.581, "p95_ms": 7.961}, "gstreamer_push": {"avg_ms": 1.111, "count": 183, "max_ms": 1.487, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.566, "count": 182, "max_ms": 84.339, "p95_ms": 75.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.232, "count": 182, "max_ms": 121.041, "p95_ms": 98.205}, "yolo11_640_fire_11class": {"avg_ms": 62.498, "count": 182, "max_ms": 91.037, "p95_ms": 84.039}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.152, "count": 182, "max_ms": 101.443, "p95_ms": 91.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.244, "count": 183, "max_ms": 174.303, "p95_ms": 152.103}, "read_interval": {"avg_ms": 139.741, "count": 182, "max_ms": 183.59, "p95_ms": 162.668}, "triton_wait": {"avg_ms": 76.558, "count": 182, "max_ms": 121.053, "p95_ms": 98.219}, "visualize": {"avg_ms": 5.237, "count": 182, "max_ms": 8.581, "p95_ms": 6.132}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1398}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5914.039, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562a9b526e00] error while decoding MB 36 18, bytestream -40", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 273, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:36:27+09:00", "last_error_summary": "[h264 @ 0x562a9b526e00] error while decoding MB 36 18, bytestream -40", "last_exit_at": "2026-07-06T18:33:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.928, "last_started_at": "2026-07-06T18:33:15+09:00", "pid": 22014, "process": {"exists": false, "pid": 22014}, "restart_count": 74, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.908, "sample_count": 2136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 167856, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:36:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T18:36:27+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T18:36:27+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T18:36:27.076439+09:00", "mtime_epoch": 1783330587.0764391, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185672}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:36:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1618.211, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.167, "pushed": 7.167, "read": 7.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.159, "count": 215, "max_ms": 9.542, "p95_ms": 7.15}, "gstreamer_push": {"avg_ms": 1.115, "count": 215, "max_ms": 1.761, "p95_ms": 1.251}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.288, "count": 215, "max_ms": 86.467, "p95_ms": 75.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.806, "count": 215, "max_ms": 120.196, "p95_ms": 99.561}, "yolo11_640_fire_11class": {"avg_ms": 61.512, "count": 215, "max_ms": 85.937, "p95_ms": 78.865}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.681, "count": 215, "max_ms": 101.584, "p95_ms": 90.699}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.682, "count": 215, "max_ms": 176.662, "p95_ms": 155.901}, "read_interval": {"avg_ms": 139.257, "count": 216, "max_ms": 186.677, "p95_ms": 165.138}, "triton_wait": {"avg_ms": 75.915, "count": 215, "max_ms": 120.209, "p95_ms": 99.573}, "visualize": {"avg_ms": 5.228, "count": 215, "max_ms": 8.706, "p95_ms": 5.847}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1398}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5912.945, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:36:27+09:00", "last_error_summary": "[h264 @ 0x562a9b526e00] error while decoding MB 36 18, bytestream -40", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 273, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:36:27+09:00", "last_error_summary": "[h264 @ 0x562a9b526e00] error while decoding MB 36 18, bytestream -40", "last_exit_at": "2026-07-06T18:36:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 192.033, "last_started_at": "2026-07-06T18:33:15+09:00", "pid": 22014, "process": {"exists": false, "pid": 22014}, "restart_count": 74, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.908, "sample_count": 2136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 167857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:36:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:36:27+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-06T18:36:27+09:00", "mjpeg": {"age_sec": 0.145, "exists": true, "mtime": "2026-07-06T18:36:27.076439+09:00", "mtime_epoch": 1783330587.0764391, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185672}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:36:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1618.211, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.202, "pushed": 7.202, "read": 7.169}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.16, "count": 215, "max_ms": 9.542, "p95_ms": 7.15}, "gstreamer_push": {"avg_ms": 1.115, "count": 216, "max_ms": 1.761, "p95_ms": 1.251}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.326, "count": 215, "max_ms": 86.467, "p95_ms": 75.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.977, "count": 215, "max_ms": 120.196, "p95_ms": 99.561}, "yolo11_640_fire_11class": {"avg_ms": 61.55, "count": 215, "max_ms": 85.937, "p95_ms": 78.865}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.734, "count": 215, "max_ms": 101.584, "p95_ms": 90.699}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.775, "count": 216, "max_ms": 176.662, "p95_ms": 155.901}, "read_interval": {"avg_ms": 139.286, "count": 215, "max_ms": 186.677, "p95_ms": 165.138}, "triton_wait": {"avg_ms": 76.085, "count": 215, "max_ms": 120.209, "p95_ms": 99.573}, "visualize": {"avg_ms": 5.228, "count": 215, "max_ms": 8.706, "p95_ms": 5.847}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1398}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5912.891, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 273, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:36:27+09:00", "last_error_summary": "[h264 @ 0x562a9b526e00] error while decoding MB 36 18, bytestream -40", "last_exit_at": "2026-07-06T18:36:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 192.038, "last_started_at": "2026-07-06T18:33:15+09:00", "pid": 22014, "process": {"exists": false, "pid": 22014}, "restart_count": 74, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.908, "sample_count": 2136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 167857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:36:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T18:36:27+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-06T18:36:27+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-06T18:36:27.076439+09:00", "mtime_epoch": 1783330587.0764391, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185672}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:36:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1618.211, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.201, "pushed": 7.201, "read": 7.167}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.16, "count": 215, "max_ms": 9.542, "p95_ms": 7.15}, "gstreamer_push": {"avg_ms": 1.115, "count": 216, "max_ms": 1.761, "p95_ms": 1.251}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.326, "count": 215, "max_ms": 86.467, "p95_ms": 75.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.977, "count": 215, "max_ms": 120.196, "p95_ms": 99.561}, "yolo11_640_fire_11class": {"avg_ms": 61.55, "count": 215, "max_ms": 85.937, "p95_ms": 78.865}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.734, "count": 215, "max_ms": 101.584, "p95_ms": 90.699}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.775, "count": 216, "max_ms": 176.662, "p95_ms": 155.901}, "read_interval": {"avg_ms": 139.286, "count": 215, "max_ms": 186.677, "p95_ms": 165.138}, "triton_wait": {"avg_ms": 76.085, "count": 215, "max_ms": 120.209, "p95_ms": 99.573}, "visualize": {"avg_ms": 5.228, "count": 215, "max_ms": 8.706, "p95_ms": 5.847}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1398}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6013.281, "limit_mb": 8796093022207.996}, "detail": {"pid": 22121, "restart_count": 75, "start_count": 76}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 273, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:36:27+09:00", "last_error_summary": "[h264 @ 0x562a9b526e00] error while decoding MB 36 18, bytestream -40", "last_exit_at": "2026-07-06T18:36:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:36:29+09:00", "pid": 22121, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.973, "name": "ffmpeg", "pid": 22121, "rss_mb": 138.973, "state": "S (sleeping)", "threads": 106}, "restart_count": 75, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.908, "sample_count": 2107, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 167857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:36:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.097, "last_frame_pushed_at": "2026-07-06T18:36:27+09:00", "last_frame_read_age_sec": 4.249, "last_frame_read_at": "2026-07-06T18:36:27+09:00", "mjpeg": {"age_sec": 4.081, "exists": true, "mtime": "2026-07-06T18:36:27.232442+09:00", "mtime_epoch": 1783330587.232442, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185810}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:36:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1618.211, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 186, "pushed_frames": 186, "read_frames": 185, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.226, "pushed": 6.226, "read": 6.192}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.161, "count": 186, "max_ms": 9.542, "p95_ms": 7.15}, "gstreamer_push": {"avg_ms": 1.114, "count": 186, "max_ms": 1.761, "p95_ms": 1.243}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.548, "count": 186, "max_ms": 86.467, "p95_ms": 75.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.891, "count": 186, "max_ms": 120.196, "p95_ms": 98.176}, "yolo11_640_fire_11class": {"avg_ms": 61.617, "count": 186, "max_ms": 85.937, "p95_ms": 79.998}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.927, "count": 186, "max_ms": 101.584, "p95_ms": 91.022}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.849, "count": 186, "max_ms": 176.662, "p95_ms": 153.751}, "read_interval": {"avg_ms": 139.331, "count": 185, "max_ms": 186.677, "p95_ms": 163.526}, "triton_wait": {"avg_ms": 76.004, "count": 186, "max_ms": 120.209, "p95_ms": 98.188}, "visualize": {"avg_ms": 5.239, "count": 186, "max_ms": 8.706, "p95_ms": 5.85}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1408}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5916.492, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5566f7147f40] error while decoding MB 45 35, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 274, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:39:12+09:00", "last_error_summary": "[h264 @ 0x5566f7147f40] error while decoding MB 45 35, bytestream -7", "last_exit_at": "2026-07-06T18:36:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.353, "last_started_at": "2026-07-06T18:36:29+09:00", "pid": 22121, "process": {"exists": false, "pid": 22121}, "restart_count": 75, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.09, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168975, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:39:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.052, "last_frame_pushed_at": "2026-07-06T18:39:12+09:00", "last_frame_read_age_sec": 0.044, "last_frame_read_at": "2026-07-06T18:39:12+09:00", "mjpeg": {"age_sec": 0.034, "exists": true, "mtime": "2026-07-06T18:39:12.631550+09:00", "mtime_epoch": 1783330752.6315503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168161}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:39:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1618.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.935, "pushed": 6.935, "read": 6.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.52, "count": 207, "max_ms": 10.143, "p95_ms": 8.446}, "gstreamer_push": {"avg_ms": 1.099, "count": 207, "max_ms": 1.403, "p95_ms": 1.237}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.365, "count": 207, "max_ms": 80.231, "p95_ms": 73.608}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.768, "count": 207, "max_ms": 116.469, "p95_ms": 100.908}, "yolo11_640_fire_11class": {"avg_ms": 61.645, "count": 207, "max_ms": 87.079, "p95_ms": 80.793}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.77, "count": 207, "max_ms": 101.168, "p95_ms": 92.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.373, "count": 207, "max_ms": 192.552, "p95_ms": 175.501}, "read_interval": {"avg_ms": 144.741, "count": 207, "max_ms": 203.594, "p95_ms": 184.301}, "triton_wait": {"avg_ms": 76.323, "count": 207, "max_ms": 116.482, "p95_ms": 101.181}, "visualize": {"avg_ms": 5.414, "count": 207, "max_ms": 7.642, "p95_ms": 6.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.849, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1408}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5915.516, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:39:12+09:00", "last_error_summary": "[h264 @ 0x5566f7147f40] error while decoding MB 45 35, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 274, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:39:12+09:00", "last_error_summary": "[h264 @ 0x5566f7147f40] error while decoding MB 45 35, bytestream -7", "last_exit_at": "2026-07-06T18:39:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.449, "last_started_at": "2026-07-06T18:36:29+09:00", "pid": 22121, "process": {"exists": false, "pid": 22121}, "restart_count": 75, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.09, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:39:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T18:39:12+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-06T18:39:12+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T18:39:12.631550+09:00", "mtime_epoch": 1783330752.6315503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168161}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:39:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1618.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.946, "pushed": 6.946, "read": 6.913}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.522, "count": 208, "max_ms": 10.143, "p95_ms": 8.446}, "gstreamer_push": {"avg_ms": 1.099, "count": 208, "max_ms": 1.403, "p95_ms": 1.237}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.412, "count": 208, "max_ms": 80.231, "p95_ms": 73.608}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.774, "count": 208, "max_ms": 116.469, "p95_ms": 100.908}, "yolo11_640_fire_11class": {"avg_ms": 61.692, "count": 208, "max_ms": 87.079, "p95_ms": 80.793}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.796, "count": 208, "max_ms": 101.168, "p95_ms": 92.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.354, "count": 208, "max_ms": 192.552, "p95_ms": 175.501}, "read_interval": {"avg_ms": 144.741, "count": 207, "max_ms": 203.594, "p95_ms": 184.301}, "triton_wait": {"avg_ms": 76.326, "count": 208, "max_ms": 116.482, "p95_ms": 101.181}, "visualize": {"avg_ms": 5.415, "count": 208, "max_ms": 7.642, "p95_ms": 6.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1408}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5915.633, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 274, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:39:12+09:00", "last_error_summary": "[h264 @ 0x5566f7147f40] error while decoding MB 45 35, bytestream -7", "last_exit_at": "2026-07-06T18:39:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.454, "last_started_at": "2026-07-06T18:36:29+09:00", "pid": 22121, "process": {"exists": false, "pid": 22121}, "restart_count": 75, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.09, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:39:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T18:39:12+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-06T18:39:12+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-06T18:39:12.631550+09:00", "mtime_epoch": 1783330752.6315503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168161}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:39:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1618.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.945, "pushed": 6.945, "read": 6.911}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.522, "count": 208, "max_ms": 10.143, "p95_ms": 8.446}, "gstreamer_push": {"avg_ms": 1.099, "count": 208, "max_ms": 1.403, "p95_ms": 1.237}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.412, "count": 208, "max_ms": 80.231, "p95_ms": 73.608}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.774, "count": 208, "max_ms": 116.469, "p95_ms": 100.908}, "yolo11_640_fire_11class": {"avg_ms": 61.692, "count": 208, "max_ms": 87.079, "p95_ms": 80.793}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.796, "count": 208, "max_ms": 101.168, "p95_ms": 92.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.354, "count": 208, "max_ms": 192.552, "p95_ms": 175.501}, "read_interval": {"avg_ms": 144.741, "count": 207, "max_ms": 203.594, "p95_ms": 184.301}, "triton_wait": {"avg_ms": 76.326, "count": 208, "max_ms": 116.482, "p95_ms": 101.181}, "visualize": {"avg_ms": 5.415, "count": 208, "max_ms": 7.642, "p95_ms": 6.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1408}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6024.191, "limit_mb": 8796093022207.996}, "detail": {"pid": 22228, "restart_count": 76, "start_count": 77}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 274, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:39:12+09:00", "last_error_summary": "[h264 @ 0x5566f7147f40] error while decoding MB 45 35, bytestream -7", "last_exit_at": "2026-07-06T18:39:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:39:15+09:00", "pid": 22228, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.184, "name": "ffmpeg", "pid": 22228, "rss_mb": 139.184, "state": "S (sleeping)", "threads": 106}, "restart_count": 76, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.226, "sample_count": 2058, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:39:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.672, "last_frame_pushed_at": "2026-07-06T18:39:12+09:00", "last_frame_read_age_sec": 4.805, "last_frame_read_at": "2026-07-06T18:39:12+09:00", "mjpeg": {"age_sec": 4.654, "exists": true, "mtime": "2026-07-06T18:39:12.771553+09:00", "mtime_epoch": 1783330752.771553, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:39:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1618.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.845, "pushed": 5.845, "read": 5.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.563, "count": 174, "max_ms": 10.143, "p95_ms": 8.449}, "gstreamer_push": {"avg_ms": 1.097, "count": 175, "max_ms": 1.403, "p95_ms": 1.236}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.935, "count": 174, "max_ms": 80.231, "p95_ms": 73.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.805, "count": 174, "max_ms": 116.469, "p95_ms": 102.077}, "yolo11_640_fire_11class": {"avg_ms": 61.324, "count": 174, "max_ms": 86.813, "p95_ms": 80.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.386, "count": 174, "max_ms": 97.818, "p95_ms": 92.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.057, "count": 175, "max_ms": 192.552, "p95_ms": 176.487}, "read_interval": {"avg_ms": 145.412, "count": 174, "max_ms": 203.594, "p95_ms": 184.935}, "triton_wait": {"avg_ms": 76.282, "count": 174, "max_ms": 116.482, "p95_ms": 102.09}, "visualize": {"avg_ms": 5.425, "count": 174, "max_ms": 7.642, "p95_ms": 6.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1419}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5922.059, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559dcd3bb480] error while decoding MB 76 23, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 275, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:42:22+09:00", "last_error_summary": "[h264 @ 0x559dcd3bb480] error while decoding MB 76 23, bytestream -14", "last_exit_at": "2026-07-06T18:39:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.365, "last_started_at": "2026-07-06T18:39:15+09:00", "pid": 22228, "process": {"exists": false, "pid": 22228}, "restart_count": 76, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.209, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:42:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.049, "last_frame_pushed_at": "2026-07-06T18:42:22+09:00", "last_frame_read_age_sec": 0.04, "last_frame_read_at": "2026-07-06T18:42:22+09:00", "mjpeg": {"age_sec": 0.027, "exists": true, "mtime": "2026-07-06T18:42:22.762951+09:00", "mtime_epoch": 1783330942.762951, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187447}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:42:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1619.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.258, "pushed": 7.258, "read": 7.258}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.223, "count": 217, "max_ms": 10.087, "p95_ms": 8.463}, "gstreamer_push": {"avg_ms": 1.114, "count": 217, "max_ms": 1.561, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.004, "count": 217, "max_ms": 86.27, "p95_ms": 73.408}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.774, "count": 217, "max_ms": 114.374, "p95_ms": 102.035}, "yolo11_640_fire_11class": {"avg_ms": 61.076, "count": 217, "max_ms": 90.711, "p95_ms": 83.523}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.868, "count": 217, "max_ms": 104.472, "p95_ms": 94.72}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.494, "count": 217, "max_ms": 169.663, "p95_ms": 158.544}, "read_interval": {"avg_ms": 138.1, "count": 217, "max_ms": 181.71, "p95_ms": 167.753}, "triton_wait": {"avg_ms": 75.159, "count": 217, "max_ms": 114.388, "p95_ms": 102.047}, "visualize": {"avg_ms": 5.145, "count": 217, "max_ms": 7.733, "p95_ms": 6.201}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1419}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5921.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:42:22+09:00", "last_error_summary": "[h264 @ 0x559dcd3bb480] error while decoding MB 76 23, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 275, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:42:22+09:00", "last_error_summary": "[h264 @ 0x559dcd3bb480] error while decoding MB 76 23, bytestream -14", "last_exit_at": "2026-07-06T18:42:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.448, "last_started_at": "2026-07-06T18:39:15+09:00", "pid": 22228, "process": {"exists": false, "pid": 22228}, "restart_count": 76, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.209, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:42:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:42:22+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-06T18:42:22+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-06T18:42:22.762951+09:00", "mtime_epoch": 1783330942.762951, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187447}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:42:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1619.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.271, "pushed": 7.271, "read": 7.238}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.221, "count": 217, "max_ms": 10.087, "p95_ms": 8.463}, "gstreamer_push": {"avg_ms": 1.114, "count": 218, "max_ms": 1.561, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.973, "count": 217, "max_ms": 86.27, "p95_ms": 73.408}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.806, "count": 217, "max_ms": 114.374, "p95_ms": 102.035}, "yolo11_640_fire_11class": {"avg_ms": 61.068, "count": 217, "max_ms": 90.711, "p95_ms": 83.523}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.86, "count": 217, "max_ms": 104.472, "p95_ms": 94.72}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.436, "count": 218, "max_ms": 169.663, "p95_ms": 158.544}, "read_interval": {"avg_ms": 138.1, "count": 217, "max_ms": 181.71, "p95_ms": 167.753}, "triton_wait": {"avg_ms": 75.19, "count": 217, "max_ms": 114.388, "p95_ms": 102.047}, "visualize": {"avg_ms": 5.143, "count": 217, "max_ms": 7.733, "p95_ms": 6.201}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1419}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5921.363, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 275, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:42:22+09:00", "last_error_summary": "[h264 @ 0x559dcd3bb480] error while decoding MB 76 23, bytestream -14", "last_exit_at": "2026-07-06T18:42:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.454, "last_started_at": "2026-07-06T18:39:15+09:00", "pid": 22228, "process": {"exists": false, "pid": 22228}, "restart_count": 76, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.209, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:42:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T18:42:22+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T18:42:22+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T18:42:22.762951+09:00", "mtime_epoch": 1783330942.762951, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187447}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:42:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1619.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.27, "pushed": 7.27, "read": 7.236}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.221, "count": 217, "max_ms": 10.087, "p95_ms": 8.463}, "gstreamer_push": {"avg_ms": 1.114, "count": 218, "max_ms": 1.561, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.973, "count": 217, "max_ms": 86.27, "p95_ms": 73.408}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.806, "count": 217, "max_ms": 114.374, "p95_ms": 102.035}, "yolo11_640_fire_11class": {"avg_ms": 61.068, "count": 217, "max_ms": 90.711, "p95_ms": 83.523}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.86, "count": 217, "max_ms": 104.472, "p95_ms": 94.72}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.436, "count": 218, "max_ms": 169.663, "p95_ms": 158.544}, "read_interval": {"avg_ms": 138.1, "count": 217, "max_ms": 181.71, "p95_ms": 167.753}, "triton_wait": {"avg_ms": 75.19, "count": 217, "max_ms": 114.388, "p95_ms": 102.047}, "visualize": {"avg_ms": 5.143, "count": 217, "max_ms": 7.733, "p95_ms": 6.201}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1419}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6031.461, "limit_mb": 8796093022207.996}, "detail": {"pid": 22335, "restart_count": 77, "start_count": 78}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 275, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:42:22+09:00", "last_error_summary": "[h264 @ 0x559dcd3bb480] error while decoding MB 76 23, bytestream -14", "last_exit_at": "2026-07-06T18:42:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:42:25+09:00", "pid": 22335, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.414, "name": "ffmpeg", "pid": 22335, "rss_mb": 141.414, "state": "S (sleeping)", "threads": 106}, "restart_count": 77, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.459, "sample_count": 2075, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:42:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.252, "last_frame_pushed_at": "2026-07-06T18:42:22+09:00", "last_frame_read_age_sec": 4.369, "last_frame_read_at": "2026-07-06T18:42:22+09:00", "mjpeg": {"age_sec": 4.233, "exists": true, "mtime": "2026-07-06T18:42:22.886953+09:00", "mtime_epoch": 1783330942.886953, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179472}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:42:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1619.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 187, "pushed_frames": 187, "read_frames": 186, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.264, "pushed": 6.264, "read": 6.23}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.292, "count": 187, "max_ms": 10.087, "p95_ms": 8.858}, "gstreamer_push": {"avg_ms": 1.119, "count": 187, "max_ms": 1.561, "p95_ms": 1.254}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.543, "count": 187, "max_ms": 86.27, "p95_ms": 73.259}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.217, "count": 187, "max_ms": 114.374, "p95_ms": 101.971}, "yolo11_640_fire_11class": {"avg_ms": 60.47, "count": 187, "max_ms": 90.711, "p95_ms": 83.523}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.189, "count": 187, "max_ms": 104.472, "p95_ms": 94.621}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.203, "count": 187, "max_ms": 169.615, "p95_ms": 158.162}, "read_interval": {"avg_ms": 137.864, "count": 186, "max_ms": 181.71, "p95_ms": 167.126}, "triton_wait": {"avg_ms": 74.548, "count": 187, "max_ms": 114.388, "p95_ms": 101.984}, "visualize": {"avg_ms": 5.199, "count": 187, "max_ms": 7.733, "p95_ms": 6.215}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.854, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1430}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5919.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c984d48e40] error while decoding MB 18 42, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 276, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:45:29+09:00", "last_error_summary": "[h264 @ 0x55c984d48e40] error while decoding MB 18 42, bytestream -13", "last_exit_at": "2026-07-06T18:42:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.449, "last_started_at": "2026-07-06T18:42:25+09:00", "pid": 22335, "process": {"exists": false, "pid": 22335}, "restart_count": 77, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.185, "sample_count": 2110, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 171602, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:45:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T18:45:29+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T18:45:29+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T18:45:29.526181+09:00", "mtime_epoch": 1783331129.5261815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:45:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1620.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.878, "pushed": 6.878, "read": 6.878}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.59, "count": 205, "max_ms": 12.371, "p95_ms": 8.481}, "gstreamer_push": {"avg_ms": 1.1, "count": 206, "max_ms": 1.678, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.451, "count": 205, "max_ms": 84.112, "p95_ms": 72.008}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.159, "count": 205, "max_ms": 115.565, "p95_ms": 101.183}, "yolo11_640_fire_11class": {"avg_ms": 62.176, "count": 205, "max_ms": 91.455, "p95_ms": 82.505}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.415, "count": 205, "max_ms": 105.555, "p95_ms": 94.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.405, "count": 206, "max_ms": 181.667, "p95_ms": 167.344}, "read_interval": {"avg_ms": 145.845, "count": 206, "max_ms": 190.039, "p95_ms": 176.857}, "triton_wait": {"avg_ms": 77.404, "count": 205, "max_ms": 115.577, "p95_ms": 101.196}, "visualize": {"avg_ms": 5.439, "count": 205, "max_ms": 10.473, "p95_ms": 6.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1430}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5919.379, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:45:29+09:00", "last_error_summary": "[h264 @ 0x55c984d48e40] error while decoding MB 18 42, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 276, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:45:29+09:00", "last_error_summary": "[h264 @ 0x55c984d48e40] error while decoding MB 18 42, bytestream -13", "last_exit_at": "2026-07-06T18:45:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.516, "last_started_at": "2026-07-06T18:42:25+09:00", "pid": 22335, "process": {"exists": false, "pid": 22335}, "restart_count": 77, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.778, "sample_count": 2111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 171603, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:45:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:45:29+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-06T18:45:29+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T18:45:29.526181+09:00", "mtime_epoch": 1783331129.5261815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:45:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1620.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.902, "pushed": 6.902, "read": 6.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.611, "count": 206, "max_ms": 12.371, "p95_ms": 8.49}, "gstreamer_push": {"avg_ms": 1.1, "count": 206, "max_ms": 1.678, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.371, "count": 206, "max_ms": 84.112, "p95_ms": 72.008}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.044, "count": 206, "max_ms": 115.565, "p95_ms": 101.183}, "yolo11_640_fire_11class": {"avg_ms": 62.105, "count": 206, "max_ms": 91.455, "p95_ms": 82.505}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.309, "count": 206, "max_ms": 105.555, "p95_ms": 94.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.252, "count": 206, "max_ms": 181.667, "p95_ms": 167.344}, "read_interval": {"avg_ms": 145.801, "count": 205, "max_ms": 190.039, "p95_ms": 176.857}, "triton_wait": {"avg_ms": 77.287, "count": 206, "max_ms": 115.577, "p95_ms": 101.196}, "visualize": {"avg_ms": 5.444, "count": 206, "max_ms": 10.473, "p95_ms": 6.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1430}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5919.289, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 276, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:45:29+09:00", "last_error_summary": "[h264 @ 0x55c984d48e40] error while decoding MB 18 42, bytestream -13", "last_exit_at": "2026-07-06T18:45:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.522, "last_started_at": "2026-07-06T18:42:25+09:00", "pid": 22335, "process": {"exists": false, "pid": 22335}, "restart_count": 77, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.778, "sample_count": 2111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 171603, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:45:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T18:45:29+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T18:45:29+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T18:45:29.526181+09:00", "mtime_epoch": 1783331129.5261815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:45:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1620.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.9, "pushed": 6.9, "read": 6.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.611, "count": 206, "max_ms": 12.371, "p95_ms": 8.49}, "gstreamer_push": {"avg_ms": 1.1, "count": 206, "max_ms": 1.678, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.371, "count": 206, "max_ms": 84.112, "p95_ms": 72.008}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.044, "count": 206, "max_ms": 115.565, "p95_ms": 101.183}, "yolo11_640_fire_11class": {"avg_ms": 62.105, "count": 206, "max_ms": 91.455, "p95_ms": 82.505}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.309, "count": 206, "max_ms": 105.555, "p95_ms": 94.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.252, "count": 206, "max_ms": 181.667, "p95_ms": 167.344}, "read_interval": {"avg_ms": 145.801, "count": 205, "max_ms": 190.039, "p95_ms": 176.857}, "triton_wait": {"avg_ms": 77.287, "count": 206, "max_ms": 115.577, "p95_ms": 101.196}, "visualize": {"avg_ms": 5.444, "count": 206, "max_ms": 10.473, "p95_ms": 6.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.854, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1430}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6027.77, "limit_mb": 8796093022207.996}, "detail": {"pid": 22442, "restart_count": 78, "start_count": 79}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 276, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:45:29+09:00", "last_error_summary": "[h264 @ 0x55c984d48e40] error while decoding MB 18 42, bytestream -13", "last_exit_at": "2026-07-06T18:45:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:45:32+09:00", "pid": 22442, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.535, "name": "ffmpeg", "pid": 22442, "rss_mb": 139.535, "state": "S (sleeping)", "threads": 106}, "restart_count": 78, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.706, "sample_count": 2077, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 171603, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:45:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.695, "last_frame_pushed_at": "2026-07-06T18:45:29+09:00", "last_frame_read_age_sec": 4.812, "last_frame_read_at": "2026-07-06T18:45:29+09:00", "mjpeg": {"age_sec": 4.675, "exists": true, "mtime": "2026-07-06T18:45:29.650184+09:00", "mtime_epoch": 1783331129.6501837, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185034}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:45:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1620.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.844, "pushed": 5.844, "read": 5.811}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.585, "count": 175, "max_ms": 12.371, "p95_ms": 8.49}, "gstreamer_push": {"avg_ms": 1.105, "count": 175, "max_ms": 1.678, "p95_ms": 1.233}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.913, "count": 175, "max_ms": 80.106, "p95_ms": 72.008}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.591, "count": 175, "max_ms": 115.565, "p95_ms": 99.818}, "yolo11_640_fire_11class": {"avg_ms": 61.762, "count": 175, "max_ms": 86.445, "p95_ms": 81.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.932, "count": 175, "max_ms": 100.019, "p95_ms": 94.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.662, "count": 175, "max_ms": 179.892, "p95_ms": 167.344}, "read_interval": {"avg_ms": 145.263, "count": 174, "max_ms": 188.508, "p95_ms": 176.857}, "triton_wait": {"avg_ms": 76.846, "count": 175, "max_ms": 115.577, "p95_ms": 99.831}, "visualize": {"avg_ms": 5.437, "count": 175, "max_ms": 10.473, "p95_ms": 6.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1438}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5913.457, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556345921380] error while decoding MB 17 43, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 277, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:48:16+09:00", "last_error_summary": "[h264 @ 0x556345921380] error while decoding MB 17 43, bytestream -7", "last_exit_at": "2026-07-06T18:45:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.847, "last_started_at": "2026-07-06T18:45:32+09:00", "pid": 22442, "process": {"exists": false, "pid": 22442}, "restart_count": 78, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.569, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 172670, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:48:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.043, "last_frame_pushed_at": "2026-07-06T18:48:16+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-06T18:48:16+09:00", "mjpeg": {"age_sec": 0.027, "exists": true, "mtime": "2026-07-06T18:48:16.145109+09:00", "mtime_epoch": 1783331296.145109, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170582}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:48:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1620.531, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.304, "pushed": 6.304, "read": 6.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.146, "count": 188, "max_ms": 11.322, "p95_ms": 8.594}, "gstreamer_push": {"avg_ms": 1.05, "count": 189, "max_ms": 1.628, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.972, "count": 188, "max_ms": 92.28, "p95_ms": 72.13}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.71, "count": 188, "max_ms": 119.627, "p95_ms": 101.808}, "yolo11_640_fire_11class": {"avg_ms": 58.665, "count": 188, "max_ms": 95.194, "p95_ms": 82.556}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.761, "count": 188, "max_ms": 110.801, "p95_ms": 93.101}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.247, "count": 189, "max_ms": 199.233, "p95_ms": 184.111}, "read_interval": {"avg_ms": 159.104, "count": 189, "max_ms": 212.152, "p95_ms": 192.739}, "triton_wait": {"avg_ms": 73.893, "count": 188, "max_ms": 119.639, "p95_ms": 101.821}, "visualize": {"avg_ms": 6.265, "count": 188, "max_ms": 8.112, "p95_ms": 7.138}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1438}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5915.66, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:48:16+09:00", "last_error_summary": "[h264 @ 0x556345921380] error while decoding MB 17 43, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 277, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:48:16+09:00", "last_error_summary": "[h264 @ 0x556345921380] error while decoding MB 17 43, bytestream -7", "last_exit_at": "2026-07-06T18:48:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.979, "last_started_at": "2026-07-06T18:45:32+09:00", "pid": 22442, "process": {"exists": false, "pid": 22442}, "restart_count": 78, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.569, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 172671, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:48:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T18:48:16+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-06T18:48:16+09:00", "mjpeg": {"age_sec": 0.159, "exists": true, "mtime": "2026-07-06T18:48:16.145109+09:00", "mtime_epoch": 1783331296.145109, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170582}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:48:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1620.531, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.303, "pushed": 6.303, "read": 6.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.151, "count": 188, "max_ms": 11.322, "p95_ms": 8.594}, "gstreamer_push": {"avg_ms": 1.05, "count": 189, "max_ms": 1.628, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.054, "count": 188, "max_ms": 92.28, "p95_ms": 72.13}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.789, "count": 188, "max_ms": 119.627, "p95_ms": 101.808}, "yolo11_640_fire_11class": {"avg_ms": 58.741, "count": 188, "max_ms": 95.194, "p95_ms": 82.556}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.827, "count": 188, "max_ms": 110.801, "p95_ms": 93.101}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.428, "count": 189, "max_ms": 199.233, "p95_ms": 184.111}, "read_interval": {"avg_ms": 159.218, "count": 188, "max_ms": 212.152, "p95_ms": 192.739}, "triton_wait": {"avg_ms": 73.971, "count": 188, "max_ms": 119.639, "p95_ms": 101.821}, "visualize": {"avg_ms": 6.272, "count": 188, "max_ms": 8.112, "p95_ms": 7.138}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1438}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5915.504, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 277, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:48:16+09:00", "last_error_summary": "[h264 @ 0x556345921380] error while decoding MB 17 43, bytestream -7", "last_exit_at": "2026-07-06T18:48:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.984, "last_started_at": "2026-07-06T18:45:32+09:00", "pid": 22442, "process": {"exists": false, "pid": 22442}, "restart_count": 78, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.569, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 172671, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:48:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T18:48:16+09:00", "last_frame_read_age_sec": 0.173, "last_frame_read_at": "2026-07-06T18:48:16+09:00", "mjpeg": {"age_sec": 0.163, "exists": true, "mtime": "2026-07-06T18:48:16.145109+09:00", "mtime_epoch": 1783331296.145109, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170582}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:48:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1620.531, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.302, "pushed": 6.302, "read": 6.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.151, "count": 188, "max_ms": 11.322, "p95_ms": 8.594}, "gstreamer_push": {"avg_ms": 1.05, "count": 189, "max_ms": 1.628, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.054, "count": 188, "max_ms": 92.28, "p95_ms": 72.13}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.789, "count": 188, "max_ms": 119.627, "p95_ms": 101.808}, "yolo11_640_fire_11class": {"avg_ms": 58.741, "count": 188, "max_ms": 95.194, "p95_ms": 82.556}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.827, "count": 188, "max_ms": 110.801, "p95_ms": 93.101}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.428, "count": 189, "max_ms": 199.233, "p95_ms": 184.111}, "read_interval": {"avg_ms": 159.218, "count": 188, "max_ms": 212.152, "p95_ms": 192.739}, "triton_wait": {"avg_ms": 73.971, "count": 188, "max_ms": 119.639, "p95_ms": 101.821}, "visualize": {"avg_ms": 6.272, "count": 188, "max_ms": 8.112, "p95_ms": 7.138}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1438}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6027.242, "limit_mb": 8796093022207.996}, "detail": {"pid": 22549, "restart_count": 79, "start_count": 80}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 277, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:48:16+09:00", "last_error_summary": "[h264 @ 0x556345921380] error while decoding MB 17 43, bytestream -7", "last_exit_at": "2026-07-06T18:48:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:48:18+09:00", "pid": 22549, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.18, "name": "ffmpeg", "pid": 22549, "rss_mb": 139.18, "state": "S (sleeping)", "threads": 106}, "restart_count": 79, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.569, "sample_count": 1976, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 172671, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:48:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.516, "last_frame_pushed_at": "2026-07-06T18:48:16+09:00", "last_frame_read_age_sec": 4.681, "last_frame_read_at": "2026-07-06T18:48:16+09:00", "mjpeg": {"age_sec": 4.499, "exists": true, "mtime": "2026-07-06T18:48:16.317112+09:00", "mtime_epoch": 1783331296.317112, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171527}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:48:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1620.531, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 157, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.284, "pushed": 5.284, "read": 5.251}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.211, "count": 158, "max_ms": 11.322, "p95_ms": 8.698}, "gstreamer_push": {"avg_ms": 1.048, "count": 158, "max_ms": 1.628, "p95_ms": 1.229}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.619, "count": 158, "max_ms": 92.28, "p95_ms": 72.553}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.411, "count": 158, "max_ms": 119.627, "p95_ms": 105.715}, "yolo11_640_fire_11class": {"avg_ms": 58.794, "count": 158, "max_ms": 95.194, "p95_ms": 82.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.796, "count": 158, "max_ms": 110.801, "p95_ms": 93.101}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.723, "count": 158, "max_ms": 199.233, "p95_ms": 184.7}, "read_interval": {"avg_ms": 161.531, "count": 157, "max_ms": 212.152, "p95_ms": 194.156}, "triton_wait": {"avg_ms": 74.568, "count": 158, "max_ms": 119.639, "p95_ms": 105.729}, "visualize": {"avg_ms": 6.349, "count": 158, "max_ms": 8.112, "p95_ms": 7.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5927.391, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ae7f871280] left block unavailable for requested intra mode [h264 @ 0x55ae7f871280] error while decoding MB 0 36, bytestream 3462 [h264 @ 0x55ae7f7d9d40] cabac decode of qscale diff failed at 0 33 [h264 @ 0x55ae7f7d9d40] error while decoding MB 0 33, bytestream 7100 [h264 @ 0x55ae7f705b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ae7f705b40] error while decoding MB 0 14, bytestream 2587 [h264 @ 0x55ae7f82a440] error while decoding MB 27 32, bytestream -32 [h2", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 283, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:51:05+09:00", "last_error_summary": "[h264 @ 0x55ae7f871280] left block unavailable for requested intra mode [h264 @ 0x55ae7f871280] error while decoding MB 0 36, bytestream 3462 [h264 @ 0x55ae7f7d9d40] cabac decode of qscale diff failed at 0 33 [h264 @ 0x55ae7f7d9d40] error while decoding MB 0 33, bytestream 7100 [h264 @ 0x55ae7f705b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ae7f705b40] error while decoding MB 0 14, bytestream 2587 [h264 @ 0x55ae7f82a440] error while decoding MB 27 32, bytestream -32 [h2", "last_exit_at": "2026-07-06T18:48:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.16, "last_started_at": "2026-07-06T18:48:18+09:00", "pid": 22549, "process": {"exists": false, "pid": 22549}, "restart_count": 79, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.224, "sample_count": 1940, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173749, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:51:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T18:51:05+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-06T18:51:05+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-06T18:51:05.920123+09:00", "mtime_epoch": 1783331465.920123, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:51:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1621.047, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 208, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.942, "pushed": 6.942, "read": 6.942}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.388, "count": 207, "max_ms": 9.837, "p95_ms": 7.855}, "gstreamer_push": {"avg_ms": 1.071, "count": 208, "max_ms": 1.92, "p95_ms": 1.214}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.969, "count": 207, "max_ms": 81.075, "p95_ms": 72.947}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.537, "count": 207, "max_ms": 109.225, "p95_ms": 100.413}, "yolo11_640_fire_11class": {"avg_ms": 59.997, "count": 207, "max_ms": 87.637, "p95_ms": 80.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.659, "count": 207, "max_ms": 104.26, "p95_ms": 93.917}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.384, "count": 208, "max_ms": 188.375, "p95_ms": 175.384}, "read_interval": {"avg_ms": 144.396, "count": 208, "max_ms": 196.092, "p95_ms": 183.764}, "triton_wait": {"avg_ms": 74.05, "count": 207, "max_ms": 109.237, "p95_ms": 100.878}, "visualize": {"avg_ms": 5.524, "count": 207, "max_ms": 8.565, "p95_ms": 6.944}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5927.039, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:51:05+09:00", "last_error_summary": "[h264 @ 0x55ae7f871280] left block unavailable for requested intra mode [h264 @ 0x55ae7f871280] error while decoding MB 0 36, bytestream 3462 [h264 @ 0x55ae7f7d9d40] cabac decode of qscale diff failed at 0 33 [h264 @ 0x55ae7f7d9d40] error while decoding MB 0 33, bytestream 7100 [h264 @ 0x55ae7f705b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ae7f705b40] error while decoding MB 0 14, bytestream 2587 [h264 @ 0x55ae7f82a440] error while decoding MB 27 32, bytestream -32 [h2", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 283, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:51:05+09:00", "last_error_summary": "[h264 @ 0x55ae7f871280] left block unavailable for requested intra mode [h264 @ 0x55ae7f871280] error while decoding MB 0 36, bytestream 3462 [h264 @ 0x55ae7f7d9d40] cabac decode of qscale diff failed at 0 33 [h264 @ 0x55ae7f7d9d40] error while decoding MB 0 33, bytestream 7100 [h264 @ 0x55ae7f705b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ae7f705b40] error while decoding MB 0 14, bytestream 2587 [h264 @ 0x55ae7f82a440] error while decoding MB 27 32, bytestream -32 [h2", "last_exit_at": "2026-07-06T18:51:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.264, "last_started_at": "2026-07-06T18:48:18+09:00", "pid": 22549, "process": {"exists": false, "pid": 22549}, "restart_count": 79, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.224, "sample_count": 1940, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173750, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:51:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T18:51:06+09:00", "last_frame_read_age_sec": 0.164, "last_frame_read_at": "2026-07-06T18:51:05+09:00", "mjpeg": {"age_sec": 0.16, "exists": true, "mtime": "2026-07-06T18:51:05.920123+09:00", "mtime_epoch": 1783331465.920123, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:51:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1621.047, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.952, "pushed": 6.952, "read": 6.918}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.391, "count": 208, "max_ms": 9.837, "p95_ms": 7.855}, "gstreamer_push": {"avg_ms": 1.07, "count": 208, "max_ms": 1.92, "p95_ms": 1.214}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.03, "count": 208, "max_ms": 81.075, "p95_ms": 72.947}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.564, "count": 208, "max_ms": 109.225, "p95_ms": 100.413}, "yolo11_640_fire_11class": {"avg_ms": 60.052, "count": 208, "max_ms": 87.637, "p95_ms": 80.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.705, "count": 208, "max_ms": 104.26, "p95_ms": 93.917}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.517, "count": 208, "max_ms": 188.375, "p95_ms": 175.384}, "read_interval": {"avg_ms": 144.419, "count": 207, "max_ms": 196.092, "p95_ms": 183.764}, "triton_wait": {"avg_ms": 74.075, "count": 208, "max_ms": 109.237, "p95_ms": 100.878}, "visualize": {"avg_ms": 5.528, "count": 208, "max_ms": 8.565, "p95_ms": 6.944}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5926.984, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 283, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:51:05+09:00", "last_error_summary": "[h264 @ 0x55ae7f871280] left block unavailable for requested intra mode [h264 @ 0x55ae7f871280] error while decoding MB 0 36, bytestream 3462 [h264 @ 0x55ae7f7d9d40] cabac decode of qscale diff failed at 0 33 [h264 @ 0x55ae7f7d9d40] error while decoding MB 0 33, bytestream 7100 [h264 @ 0x55ae7f705b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ae7f705b40] error while decoding MB 0 14, bytestream 2587 [h264 @ 0x55ae7f82a440] error while decoding MB 27 32, bytestream -32 [h2", "last_exit_at": "2026-07-06T18:51:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.269, "last_started_at": "2026-07-06T18:48:18+09:00", "pid": 22549, "process": {"exists": false, "pid": 22549}, "restart_count": 79, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.224, "sample_count": 1940, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173750, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:51:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T18:51:06+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-06T18:51:05+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-06T18:51:05.920123+09:00", "mtime_epoch": 1783331465.920123, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:51:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1621.047, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.951, "pushed": 6.951, "read": 6.917}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.391, "count": 208, "max_ms": 9.837, "p95_ms": 7.855}, "gstreamer_push": {"avg_ms": 1.07, "count": 208, "max_ms": 1.92, "p95_ms": 1.214}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.03, "count": 208, "max_ms": 81.075, "p95_ms": 72.947}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.564, "count": 208, "max_ms": 109.225, "p95_ms": 100.413}, "yolo11_640_fire_11class": {"avg_ms": 60.052, "count": 208, "max_ms": 87.637, "p95_ms": 80.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.705, "count": 208, "max_ms": 104.26, "p95_ms": 93.917}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.517, "count": 208, "max_ms": 188.375, "p95_ms": 175.384}, "read_interval": {"avg_ms": 144.419, "count": 207, "max_ms": 196.092, "p95_ms": 183.764}, "triton_wait": {"avg_ms": 74.075, "count": 208, "max_ms": 109.237, "p95_ms": 100.878}, "visualize": {"avg_ms": 5.528, "count": 208, "max_ms": 8.565, "p95_ms": 6.944}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6034.957, "limit_mb": 8796093022207.996}, "detail": {"pid": 22656, "restart_count": 80, "start_count": 81}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 283, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:51:05+09:00", "last_error_summary": "[h264 @ 0x55ae7f871280] left block unavailable for requested intra mode [h264 @ 0x55ae7f871280] error while decoding MB 0 36, bytestream 3462 [h264 @ 0x55ae7f7d9d40] cabac decode of qscale diff failed at 0 33 [h264 @ 0x55ae7f7d9d40] error while decoding MB 0 33, bytestream 7100 [h264 @ 0x55ae7f705b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ae7f705b40] error while decoding MB 0 14, bytestream 2587 [h264 @ 0x55ae7f82a440] error while decoding MB 27 32, bytestream -32 [h2", "last_exit_at": "2026-07-06T18:51:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:51:08+09:00", "pid": 22656, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.27, "name": "ffmpeg", "pid": 22656, "rss_mb": 139.27, "state": "S (sleeping)", "threads": 106}, "restart_count": 80, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.398, "sample_count": 1911, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173750, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:51:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.244, "last_frame_pushed_at": "2026-07-06T18:51:06+09:00", "last_frame_read_age_sec": 4.404, "last_frame_read_at": "2026-07-06T18:51:05+09:00", "mjpeg": {"age_sec": 4.228, "exists": true, "mtime": "2026-07-06T18:51:06.092126+09:00", "mtime_epoch": 1783331466.0921261, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168061}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:51:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1621.047, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.92, "pushed": 5.92, "read": 5.886}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.467, "count": 177, "max_ms": 9.837, "p95_ms": 8.057}, "gstreamer_push": {"avg_ms": 1.066, "count": 177, "max_ms": 1.92, "p95_ms": 1.221}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.112, "count": 177, "max_ms": 81.075, "p95_ms": 72.947}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.297, "count": 177, "max_ms": 109.225, "p95_ms": 100.413}, "yolo11_640_fire_11class": {"avg_ms": 59.681, "count": 177, "max_ms": 86.997, "p95_ms": 81.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.21, "count": 177, "max_ms": 104.132, "p95_ms": 93.917}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.788, "count": 177, "max_ms": 188.375, "p95_ms": 176.296}, "read_interval": {"avg_ms": 145.62, "count": 176, "max_ms": 196.092, "p95_ms": 185.466}, "triton_wait": {"avg_ms": 73.841, "count": 177, "max_ms": 109.237, "p95_ms": 100.59}, "visualize": {"avg_ms": 5.606, "count": 177, "max_ms": 8.565, "p95_ms": 6.973}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5928.625, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ff8d096480] error while decoding MB 18 37, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 284, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:53:51+09:00", "last_error_summary": "[h264 @ 0x55ff8d096480] error while decoding MB 18 37, bytestream -8", "last_exit_at": "2026-07-06T18:51:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.181, "last_started_at": "2026-07-06T18:51:08+09:00", "pid": 22656, "process": {"exists": false, "pid": 22656}, "restart_count": 80, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.919, "sample_count": 1987, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 174857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:53:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-06T18:53:51+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-06T18:53:51+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T18:53:51.451084+09:00", "mtime_epoch": 1783331631.451084, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:53:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1621.562, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.11, "pushed": 7.11, "read": 7.11}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.385, "count": 213, "max_ms": 9.161, "p95_ms": 7.751}, "gstreamer_push": {"avg_ms": 1.096, "count": 213, "max_ms": 1.623, "p95_ms": 1.229}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.851, "count": 213, "max_ms": 82.971, "p95_ms": 78.002}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.387, "count": 213, "max_ms": 121.947, "p95_ms": 100.416}, "yolo11_640_fire_11class": {"avg_ms": 61.051, "count": 213, "max_ms": 91.744, "p95_ms": 83.084}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.503, "count": 213, "max_ms": 109.128, "p95_ms": 96.311}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.49, "count": 213, "max_ms": 182.232, "p95_ms": 163.254}, "read_interval": {"avg_ms": 140.898, "count": 213, "max_ms": 190.306, "p95_ms": 172.621}, "triton_wait": {"avg_ms": 75.733, "count": 213, "max_ms": 121.96, "p95_ms": 100.539}, "visualize": {"avg_ms": 5.309, "count": 213, "max_ms": 8.075, "p95_ms": 6.479}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5928.23, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:53:51+09:00", "last_error_summary": "[h264 @ 0x55ff8d096480] error while decoding MB 18 37, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 284, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:53:51+09:00", "last_error_summary": "[h264 @ 0x55ff8d096480] error while decoding MB 18 37, bytestream -8", "last_exit_at": "2026-07-06T18:53:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.252, "last_started_at": "2026-07-06T18:51:08+09:00", "pid": 22656, "process": {"exists": false, "pid": 22656}, "restart_count": 80, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.919, "sample_count": 1987, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 174858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:53:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:53:51+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T18:53:51+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T18:53:51.451084+09:00", "mtime_epoch": 1783331631.451084, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:53:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1621.562, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 212, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.131, "pushed": 7.131, "read": 7.098}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.43, "count": 213, "max_ms": 15.415, "p95_ms": 7.996}, "gstreamer_push": {"avg_ms": 1.097, "count": 213, "max_ms": 1.623, "p95_ms": 1.229}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.849, "count": 213, "max_ms": 82.971, "p95_ms": 78.002}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.365, "count": 213, "max_ms": 121.947, "p95_ms": 100.416}, "yolo11_640_fire_11class": {"avg_ms": 61.03, "count": 213, "max_ms": 91.744, "p95_ms": 83.084}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.475, "count": 213, "max_ms": 109.128, "p95_ms": 96.311}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.526, "count": 213, "max_ms": 182.232, "p95_ms": 163.254}, "read_interval": {"avg_ms": 140.983, "count": 212, "max_ms": 190.306, "p95_ms": 172.621}, "triton_wait": {"avg_ms": 75.71, "count": 213, "max_ms": 121.96, "p95_ms": 100.539}, "visualize": {"avg_ms": 5.314, "count": 213, "max_ms": 8.075, "p95_ms": 6.479}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5928.348, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 284, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:53:51+09:00", "last_error_summary": "[h264 @ 0x55ff8d096480] error while decoding MB 18 37, bytestream -8", "last_exit_at": "2026-07-06T18:53:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.257, "last_started_at": "2026-07-06T18:51:08+09:00", "pid": 22656, "process": {"exists": false, "pid": 22656}, "restart_count": 80, "restart_count_per_hour": 18, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.919, "sample_count": 1987, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 174858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:53:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T18:53:51+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T18:53:51+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-06T18:53:51.451084+09:00", "mtime_epoch": 1783331631.451084, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:53:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1621.562, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 212, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.13, "pushed": 7.13, "read": 7.097}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.43, "count": 213, "max_ms": 15.415, "p95_ms": 7.996}, "gstreamer_push": {"avg_ms": 1.097, "count": 213, "max_ms": 1.623, "p95_ms": 1.229}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.849, "count": 213, "max_ms": 82.971, "p95_ms": 78.002}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.365, "count": 213, "max_ms": 121.947, "p95_ms": 100.416}, "yolo11_640_fire_11class": {"avg_ms": 61.03, "count": 213, "max_ms": 91.744, "p95_ms": 83.084}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.475, "count": 213, "max_ms": 109.128, "p95_ms": 96.311}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.526, "count": 213, "max_ms": 182.232, "p95_ms": 163.254}, "read_interval": {"avg_ms": 140.983, "count": 212, "max_ms": 190.306, "p95_ms": 172.621}, "triton_wait": {"avg_ms": 75.71, "count": 213, "max_ms": 121.96, "p95_ms": 100.539}, "visualize": {"avg_ms": 5.314, "count": 213, "max_ms": 8.075, "p95_ms": 6.479}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6035.863, "limit_mb": 8796093022207.996}, "detail": {"pid": 22763, "restart_count": 81, "start_count": 82}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 284, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:53:51+09:00", "last_error_summary": "[h264 @ 0x55ff8d096480] error while decoding MB 18 37, bytestream -8", "last_exit_at": "2026-07-06T18:53:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:53:54+09:00", "pid": 22763, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.93, "name": "ffmpeg", "pid": 22763, "rss_mb": 138.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 81, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.941, "sample_count": 1954, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 174858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:53:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.837, "last_frame_pushed_at": "2026-07-06T18:53:51+09:00", "last_frame_read_age_sec": 4.961, "last_frame_read_at": "2026-07-06T18:53:51+09:00", "mjpeg": {"age_sec": 4.82, "exists": true, "mtime": "2026-07-06T18:53:51.583086+09:00", "mtime_epoch": 1783331631.5830863, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162341}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:53:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1621.562, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.951, "pushed": 5.951, "read": 5.918}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.473, "count": 178, "max_ms": 15.415, "p95_ms": 8.3}, "gstreamer_push": {"avg_ms": 1.102, "count": 178, "max_ms": 1.623, "p95_ms": 1.23}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.142, "count": 178, "max_ms": 82.971, "p95_ms": 77.093}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.619, "count": 178, "max_ms": 121.947, "p95_ms": 100.416}, "yolo11_640_fire_11class": {"avg_ms": 61.331, "count": 178, "max_ms": 90.668, "p95_ms": 82.92}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.002, "count": 178, "max_ms": 109.128, "p95_ms": 95.364}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.044, "count": 178, "max_ms": 182.232, "p95_ms": 164.213}, "read_interval": {"avg_ms": 141.563, "count": 177, "max_ms": 190.306, "p95_ms": 172.662}, "triton_wait": {"avg_ms": 75.969, "count": 178, "max_ms": 121.96, "p95_ms": 100.539}, "visualize": {"avg_ms": 5.346, "count": 178, "max_ms": 8.075, "p95_ms": 6.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1468}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5933.844, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ec472e9cc0] error while decoding MB 8 17, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 285, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:57:00+09:00", "last_error_summary": "[h264 @ 0x55ec472e9cc0] error while decoding MB 8 17, bytestream -11", "last_exit_at": "2026-07-06T18:53:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.973, "last_started_at": "2026-07-06T18:53:54+09:00", "pid": 22763, "process": {"exists": false, "pid": 22763}, "restart_count": 81, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.77, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 176180, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:57:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T18:57:00+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T18:57:00+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T18:57:00.330481+09:00", "mtime_epoch": 1783331820.330481, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:57:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1622.078, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 222, "pushed_frames": 222, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.419, "pushed": 7.419, "read": 7.419}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.083, "count": 222, "max_ms": 9.659, "p95_ms": 7.896}, "gstreamer_push": {"avg_ms": 1.081, "count": 222, "max_ms": 1.657, "p95_ms": 1.213}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.4, "count": 222, "max_ms": 79.736, "p95_ms": 73.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.081, "count": 222, "max_ms": 117.546, "p95_ms": 96.954}, "yolo11_640_fire_11class": {"avg_ms": 60.077, "count": 222, "max_ms": 89.164, "p95_ms": 80.794}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.624, "count": 222, "max_ms": 105.121, "p95_ms": 91.43}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.558, "count": 222, "max_ms": 175.694, "p95_ms": 152.483}, "read_interval": {"avg_ms": 135.048, "count": 222, "max_ms": 184.704, "p95_ms": 161.653}, "triton_wait": {"avg_ms": 73.51, "count": 222, "max_ms": 117.559, "p95_ms": 97.361}, "visualize": {"avg_ms": 5.028, "count": 222, "max_ms": 8.091, "p95_ms": 6.324}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1468}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5933.113, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T18:57:00+09:00", "last_error_summary": "[h264 @ 0x55ec472e9cc0] error while decoding MB 8 17, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 285, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:57:00+09:00", "last_error_summary": "[h264 @ 0x55ec472e9cc0] error while decoding MB 8 17, bytestream -11", "last_exit_at": "2026-07-06T18:57:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 186.043, "last_started_at": "2026-07-06T18:53:54+09:00", "pid": 22763, "process": {"exists": false, "pid": 22763}, "restart_count": 81, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.747, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 176181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:57:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T18:57:00+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-06T18:57:00+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T18:57:00.330481+09:00", "mtime_epoch": 1783331820.330481, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:57:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1622.078, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.435, "pushed": 7.435, "read": 7.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.091, "count": 222, "max_ms": 9.659, "p95_ms": 7.896}, "gstreamer_push": {"avg_ms": 1.08, "count": 223, "max_ms": 1.657, "p95_ms": 1.213}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.41, "count": 222, "max_ms": 79.736, "p95_ms": 73.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.087, "count": 222, "max_ms": 117.546, "p95_ms": 96.954}, "yolo11_640_fire_11class": {"avg_ms": 60.111, "count": 222, "max_ms": 89.164, "p95_ms": 80.794}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.661, "count": 222, "max_ms": 105.121, "p95_ms": 91.43}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.522, "count": 223, "max_ms": 175.694, "p95_ms": 152.483}, "read_interval": {"avg_ms": 135.048, "count": 222, "max_ms": 184.704, "p95_ms": 161.653}, "triton_wait": {"avg_ms": 73.516, "count": 222, "max_ms": 117.559, "p95_ms": 97.361}, "visualize": {"avg_ms": 5.038, "count": 222, "max_ms": 8.091, "p95_ms": 6.345}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1468}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5932.984, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 285, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:57:00+09:00", "last_error_summary": "[h264 @ 0x55ec472e9cc0] error while decoding MB 8 17, bytestream -11", "last_exit_at": "2026-07-06T18:57:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 186.049, "last_started_at": "2026-07-06T18:53:54+09:00", "pid": 22763, "process": {"exists": false, "pid": 22763}, "restart_count": 81, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.747, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 176181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:57:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T18:57:00+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T18:57:00+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T18:57:00.330481+09:00", "mtime_epoch": 1783331820.330481, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:57:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1622.078, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.434, "pushed": 7.434, "read": 7.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.091, "count": 222, "max_ms": 9.659, "p95_ms": 7.896}, "gstreamer_push": {"avg_ms": 1.08, "count": 223, "max_ms": 1.657, "p95_ms": 1.213}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.41, "count": 222, "max_ms": 79.736, "p95_ms": 73.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.087, "count": 222, "max_ms": 117.546, "p95_ms": 96.954}, "yolo11_640_fire_11class": {"avg_ms": 60.111, "count": 222, "max_ms": 89.164, "p95_ms": 80.794}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.661, "count": 222, "max_ms": 105.121, "p95_ms": 91.43}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.522, "count": 223, "max_ms": 175.694, "p95_ms": 152.483}, "read_interval": {"avg_ms": 135.048, "count": 222, "max_ms": 184.704, "p95_ms": 161.653}, "triton_wait": {"avg_ms": 73.516, "count": 222, "max_ms": 117.559, "p95_ms": 97.361}, "visualize": {"avg_ms": 5.038, "count": 222, "max_ms": 8.091, "p95_ms": 6.345}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1468}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6041.523, "limit_mb": 8796093022207.996}, "detail": {"pid": 22870, "restart_count": 82, "start_count": 83}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 285, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T18:57:00+09:00", "last_error_summary": "[h264 @ 0x55ec472e9cc0] error while decoding MB 8 17, bytestream -11", "last_exit_at": "2026-07-06T18:57:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T18:57:03+09:00", "pid": 22870, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.168, "name": "ffmpeg", "pid": 22870, "rss_mb": 139.168, "state": "S (sleeping)", "threads": 106}, "restart_count": 82, "restart_count_per_hour": 20, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.766, "sample_count": 2060, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 176181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T18:57:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.957, "last_frame_pushed_at": "2026-07-06T18:57:00+09:00", "last_frame_read_age_sec": 5.077, "last_frame_read_at": "2026-07-06T18:57:00+09:00", "mjpeg": {"age_sec": 4.94, "exists": true, "mtime": "2026-07-06T18:57:00.458483+09:00", "mtime_epoch": 1783331820.4584835, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170631}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T18:57:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1622.078, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 187, "pushed_frames": 187, "read_frames": 186, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.241, "pushed": 6.241, "read": 6.208}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.058, "count": 186, "max_ms": 9.431, "p95_ms": 7.896}, "gstreamer_push": {"avg_ms": 1.077, "count": 187, "max_ms": 1.657, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.058, "count": 186, "max_ms": 78.234, "p95_ms": 71.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.814, "count": 186, "max_ms": 117.546, "p95_ms": 95.828}, "yolo11_640_fire_11class": {"avg_ms": 59.878, "count": 186, "max_ms": 89.164, "p95_ms": 80.794}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.36, "count": 186, "max_ms": 105.121, "p95_ms": 91.093}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.902, "count": 187, "max_ms": 175.694, "p95_ms": 149.738}, "read_interval": {"avg_ms": 134.419, "count": 186, "max_ms": 184.704, "p95_ms": 159.592}, "triton_wait": {"avg_ms": 73.153, "count": 186, "max_ms": 117.559, "p95_ms": 95.868}, "visualize": {"avg_ms": 5.015, "count": 186, "max_ms": 8.091, "p95_ms": 6.37}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1479}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5936.426, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562ff4bd76c0] error while decoding MB 60 17, bytestream -20", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 286, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:00:08+09:00", "last_error_summary": "[h264 @ 0x562ff4bd76c0] error while decoding MB 60 17, bytestream -20", "last_exit_at": "2026-07-06T18:57:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.578, "last_started_at": "2026-07-06T18:57:03+09:00", "pid": 22870, "process": {"exists": false, "pid": 22870}, "restart_count": 82, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.148, "sample_count": 2117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 177493, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:00:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T19:00:08+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T19:00:08+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-06T19:00:08.937887+09:00", "mtime_epoch": 1783332008.9378874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:00:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1622.852, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.177, "pushed": 7.177, "read": 7.177}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.311, "count": 215, "max_ms": 9.328, "p95_ms": 7.805}, "gstreamer_push": {"avg_ms": 1.095, "count": 215, "max_ms": 1.553, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.483, "count": 215, "max_ms": 86.097, "p95_ms": 75.403}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.469, "count": 215, "max_ms": 111.653, "p95_ms": 100.449}, "yolo11_640_fire_11class": {"avg_ms": 62.018, "count": 215, "max_ms": 99.246, "p95_ms": 81.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.582, "count": 215, "max_ms": 104.141, "p95_ms": 94.689}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.982, "count": 215, "max_ms": 200.378, "p95_ms": 163.574}, "read_interval": {"avg_ms": 139.643, "count": 215, "max_ms": 208.832, "p95_ms": 172.073}, "triton_wait": {"avg_ms": 74.815, "count": 215, "max_ms": 111.665, "p95_ms": 100.461}, "visualize": {"avg_ms": 5.265, "count": 215, "max_ms": 8.022, "p95_ms": 6.362}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1479}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5935.641, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:00:08+09:00", "last_error_summary": "[h264 @ 0x562ff4bd76c0] error while decoding MB 60 17, bytestream -20", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 286, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:00:08+09:00", "last_error_summary": "[h264 @ 0x562ff4bd76c0] error while decoding MB 60 17, bytestream -20", "last_exit_at": "2026-07-06T19:00:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.675, "last_started_at": "2026-07-06T18:57:03+09:00", "pid": 22870, "process": {"exists": false, "pid": 22870}, "restart_count": 82, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.148, "sample_count": 2117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 177494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:00:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T19:00:09+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-06T19:00:08+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-06T19:00:08.937887+09:00", "mtime_epoch": 1783332008.9378874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:00:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1622.852, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.182, "pushed": 7.182, "read": 7.149}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.315, "count": 215, "max_ms": 9.328, "p95_ms": 7.805}, "gstreamer_push": {"avg_ms": 1.094, "count": 215, "max_ms": 1.553, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.467, "count": 215, "max_ms": 86.097, "p95_ms": 75.403}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.45, "count": 215, "max_ms": 111.653, "p95_ms": 100.449}, "yolo11_640_fire_11class": {"avg_ms": 62.012, "count": 215, "max_ms": 99.246, "p95_ms": 81.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.567, "count": 215, "max_ms": 104.141, "p95_ms": 94.689}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.103, "count": 215, "max_ms": 200.378, "p95_ms": 163.574}, "read_interval": {"avg_ms": 139.715, "count": 214, "max_ms": 208.832, "p95_ms": 172.073}, "triton_wait": {"avg_ms": 74.796, "count": 215, "max_ms": 111.665, "p95_ms": 100.461}, "visualize": {"avg_ms": 5.27, "count": 215, "max_ms": 8.022, "p95_ms": 6.362}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1479}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5935.633, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 286, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:00:08+09:00", "last_error_summary": "[h264 @ 0x562ff4bd76c0] error while decoding MB 60 17, bytestream -20", "last_exit_at": "2026-07-06T19:00:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.681, "last_started_at": "2026-07-06T18:57:03+09:00", "pid": 22870, "process": {"exists": false, "pid": 22870}, "restart_count": 82, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.148, "sample_count": 2117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 177494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:00:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T19:00:09+09:00", "last_frame_read_age_sec": 0.153, "last_frame_read_at": "2026-07-06T19:00:08+09:00", "mjpeg": {"age_sec": 0.141, "exists": true, "mtime": "2026-07-06T19:00:08.937887+09:00", "mtime_epoch": 1783332008.9378874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:00:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1622.852, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.181, "pushed": 7.181, "read": 7.147}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.315, "count": 215, "max_ms": 9.328, "p95_ms": 7.805}, "gstreamer_push": {"avg_ms": 1.094, "count": 215, "max_ms": 1.553, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.467, "count": 215, "max_ms": 86.097, "p95_ms": 75.403}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.45, "count": 215, "max_ms": 111.653, "p95_ms": 100.449}, "yolo11_640_fire_11class": {"avg_ms": 62.012, "count": 215, "max_ms": 99.246, "p95_ms": 81.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.567, "count": 215, "max_ms": 104.141, "p95_ms": 94.689}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.103, "count": 215, "max_ms": 200.378, "p95_ms": 163.574}, "read_interval": {"avg_ms": 139.715, "count": 214, "max_ms": 208.832, "p95_ms": 172.073}, "triton_wait": {"avg_ms": 74.796, "count": 215, "max_ms": 111.665, "p95_ms": 100.461}, "visualize": {"avg_ms": 5.27, "count": 215, "max_ms": 8.022, "p95_ms": 6.362}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1479}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6043.488, "limit_mb": 8796093022207.996}, "detail": {"pid": 22977, "restart_count": 83, "start_count": 84}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 286, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:00:08+09:00", "last_error_summary": "[h264 @ 0x562ff4bd76c0] error while decoding MB 60 17, bytestream -20", "last_exit_at": "2026-07-06T19:00:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:00:11+09:00", "pid": 22977, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.035, "name": "ffmpeg", "pid": 22977, "rss_mb": 139.035, "state": "S (sleeping)", "threads": 106}, "restart_count": 83, "restart_count_per_hour": 20, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.998, "sample_count": 2087, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 177494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:00:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.32, "last_frame_pushed_at": "2026-07-06T19:00:09+09:00", "last_frame_read_age_sec": 4.463, "last_frame_read_at": "2026-07-06T19:00:08+09:00", "mjpeg": {"age_sec": 4.303, "exists": true, "mtime": "2026-07-06T19:00:09.085890+09:00", "mtime_epoch": 1783332009.08589, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169736}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:00:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1622.852, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.138, "pushed": 6.138, "read": 6.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.363, "count": 183, "max_ms": 9.328, "p95_ms": 7.85}, "gstreamer_push": {"avg_ms": 1.094, "count": 184, "max_ms": 1.553, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.303, "count": 183, "max_ms": 86.097, "p95_ms": 74.811}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.174, "count": 183, "max_ms": 111.653, "p95_ms": 97.246}, "yolo11_640_fire_11class": {"avg_ms": 61.897, "count": 183, "max_ms": 99.246, "p95_ms": 82.129}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.325, "count": 183, "max_ms": 104.141, "p95_ms": 93.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.665, "count": 184, "max_ms": 200.378, "p95_ms": 163.574}, "read_interval": {"avg_ms": 140.164, "count": 183, "max_ms": 208.832, "p95_ms": 172.073}, "triton_wait": {"avg_ms": 74.528, "count": 183, "max_ms": 111.665, "p95_ms": 97.968}, "visualize": {"avg_ms": 5.325, "count": 183, "max_ms": 8.022, "p95_ms": 6.41}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5934.859, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55643cb92200] error while decoding MB 46 2, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 287, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:03:13+09:00", "last_error_summary": "[h264 @ 0x55643cb92200] error while decoding MB 46 2, bytestream -12", "last_exit_at": "2026-07-06T19:00:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 181.725, "last_started_at": "2026-07-06T19:00:11+09:00", "pid": 22977, "process": {"exists": false, "pid": 22977}, "restart_count": 83, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.188, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 178766, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:03:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.049, "last_frame_pushed_at": "2026-07-06T19:03:13+09:00", "last_frame_read_age_sec": 0.043, "last_frame_read_at": "2026-07-06T19:03:13+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-06T19:03:13.081223+09:00", "mtime_epoch": 1783332193.0812225, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:03:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1623.367, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.116, "pushed": 7.116, "read": 7.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.56, "count": 213, "max_ms": 10.289, "p95_ms": 9.285}, "gstreamer_push": {"avg_ms": 1.082, "count": 213, "max_ms": 1.377, "p95_ms": 1.211}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.118, "count": 213, "max_ms": 84.092, "p95_ms": 73.657}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.654, "count": 213, "max_ms": 115.173, "p95_ms": 98.523}, "yolo11_640_fire_11class": {"avg_ms": 60.127, "count": 213, "max_ms": 87.745, "p95_ms": 83.602}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.364, "count": 213, "max_ms": 104.76, "p95_ms": 93.308}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.145, "count": 213, "max_ms": 173.642, "p95_ms": 161.407}, "read_interval": {"avg_ms": 140.868, "count": 213, "max_ms": 185.484, "p95_ms": 170.152}, "triton_wait": {"avg_ms": 74.134, "count": 213, "max_ms": 115.185, "p95_ms": 98.977}, "visualize": {"avg_ms": 5.47, "count": 213, "max_ms": 8.972, "p95_ms": 7.159}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5933.598, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:03:13+09:00", "last_error_summary": "[h264 @ 0x55643cb92200] error while decoding MB 46 2, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 287, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:03:13+09:00", "last_error_summary": "[h264 @ 0x55643cb92200] error while decoding MB 46 2, bytestream -12", "last_exit_at": "2026-07-06T19:03:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 181.856, "last_started_at": "2026-07-06T19:00:11+09:00", "pid": 22977, "process": {"exists": false, "pid": 22977}, "restart_count": 83, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.192, "sample_count": 2103, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 178767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:03:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T19:03:13+09:00", "last_frame_read_age_sec": 0.173, "last_frame_read_at": "2026-07-06T19:03:13+09:00", "mjpeg": {"age_sec": 0.162, "exists": true, "mtime": "2026-07-06T19:03:13.081223+09:00", "mtime_epoch": 1783332193.0812225, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:03:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1623.367, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 212, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.122, "pushed": 7.122, "read": 7.088}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.561, "count": 213, "max_ms": 10.289, "p95_ms": 9.285}, "gstreamer_push": {"avg_ms": 1.081, "count": 213, "max_ms": 1.377, "p95_ms": 1.211}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.24, "count": 213, "max_ms": 84.092, "p95_ms": 73.657}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.77, "count": 213, "max_ms": 115.173, "p95_ms": 98.523}, "yolo11_640_fire_11class": {"avg_ms": 60.235, "count": 213, "max_ms": 87.745, "p95_ms": 83.602}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.487, "count": 213, "max_ms": 104.76, "p95_ms": 93.308}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.379, "count": 213, "max_ms": 173.642, "p95_ms": 161.519}, "read_interval": {"avg_ms": 140.939, "count": 212, "max_ms": 185.484, "p95_ms": 170.152}, "triton_wait": {"avg_ms": 74.25, "count": 213, "max_ms": 115.185, "p95_ms": 98.977}, "visualize": {"avg_ms": 5.476, "count": 213, "max_ms": 8.972, "p95_ms": 7.159}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5933.574, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 287, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:03:13+09:00", "last_error_summary": "[h264 @ 0x55643cb92200] error while decoding MB 46 2, bytestream -12", "last_exit_at": "2026-07-06T19:03:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 181.86, "last_started_at": "2026-07-06T19:00:11+09:00", "pid": 22977, "process": {"exists": false, "pid": 22977}, "restart_count": 83, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.192, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 178767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:03:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T19:03:13+09:00", "last_frame_read_age_sec": 0.177, "last_frame_read_at": "2026-07-06T19:03:13+09:00", "mjpeg": {"age_sec": 0.166, "exists": true, "mtime": "2026-07-06T19:03:13.081223+09:00", "mtime_epoch": 1783332193.0812225, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:03:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1623.367, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 212, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.121, "pushed": 7.121, "read": 7.087}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.561, "count": 213, "max_ms": 10.289, "p95_ms": 9.285}, "gstreamer_push": {"avg_ms": 1.081, "count": 213, "max_ms": 1.377, "p95_ms": 1.211}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.24, "count": 213, "max_ms": 84.092, "p95_ms": 73.657}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.77, "count": 213, "max_ms": 115.173, "p95_ms": 98.523}, "yolo11_640_fire_11class": {"avg_ms": 60.235, "count": 213, "max_ms": 87.745, "p95_ms": 83.602}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.487, "count": 213, "max_ms": 104.76, "p95_ms": 93.308}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.379, "count": 213, "max_ms": 173.642, "p95_ms": 161.519}, "read_interval": {"avg_ms": 140.939, "count": 212, "max_ms": 185.484, "p95_ms": 170.152}, "triton_wait": {"avg_ms": 74.25, "count": 213, "max_ms": 115.185, "p95_ms": 98.977}, "visualize": {"avg_ms": 5.476, "count": 213, "max_ms": 8.972, "p95_ms": 7.159}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6042.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 23084, "restart_count": 84, "start_count": 85}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 287, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:03:13+09:00", "last_error_summary": "[h264 @ 0x55643cb92200] error while decoding MB 46 2, bytestream -12", "last_exit_at": "2026-07-06T19:03:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:03:15+09:00", "pid": 23084, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.051, "name": "ffmpeg", "pid": 23084, "rss_mb": 139.051, "state": "S (sleeping)", "threads": 106}, "restart_count": 84, "restart_count_per_hour": 20, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.188, "sample_count": 2070, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 178767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:03:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.68, "last_frame_pushed_at": "2026-07-06T19:03:13+09:00", "last_frame_read_age_sec": 4.848, "last_frame_read_at": "2026-07-06T19:03:13+09:00", "mjpeg": {"age_sec": 4.662, "exists": true, "mtime": "2026-07-06T19:03:13.257226+09:00", "mtime_epoch": 1783332193.2572255, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169391}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:03:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1623.367, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.022, "pushed": 6.022, "read": 5.989}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.609, "count": 180, "max_ms": 10.289, "p95_ms": 9.086}, "gstreamer_push": {"avg_ms": 1.076, "count": 180, "max_ms": 1.377, "p95_ms": 1.2}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.904, "count": 180, "max_ms": 83.861, "p95_ms": 73.502}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.953, "count": 180, "max_ms": 110.134, "p95_ms": 98.441}, "yolo11_640_fire_11class": {"avg_ms": 59.768, "count": 180, "max_ms": 87.175, "p95_ms": 81.619}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.93, "count": 180, "max_ms": 103.131, "p95_ms": 92.495}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.153, "count": 180, "max_ms": 173.642, "p95_ms": 161.519}, "read_interval": {"avg_ms": 140.548, "count": 179, "max_ms": 182.891, "p95_ms": 170.434}, "triton_wait": {"avg_ms": 73.441, "count": 180, "max_ms": 110.146, "p95_ms": 98.535}, "visualize": {"avg_ms": 5.498, "count": 180, "max_ms": 8.64, "p95_ms": 6.991}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1499}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5934.145, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55db3d26ae00] error while decoding MB 35 16, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 288, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:06:13+09:00", "last_error_summary": "[h264 @ 0x55db3d26ae00] error while decoding MB 35 16, bytestream -5", "last_exit_at": "2026-07-06T19:03:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 177.696, "last_started_at": "2026-07-06T19:03:15+09:00", "pid": 23084, "process": {"exists": false, "pid": 23084}, "restart_count": 84, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.253, "sample_count": 2067, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 179990, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:06:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.053, "last_frame_pushed_at": "2026-07-06T19:06:13+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-06T19:06:13+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-06T19:06:13.576498+09:00", "mtime_epoch": 1783332373.5764978, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158239}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:06:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.141, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.809, "pushed": 6.809, "read": 6.809}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.723, "count": 204, "max_ms": 10.823, "p95_ms": 8.102}, "gstreamer_push": {"avg_ms": 1.072, "count": 204, "max_ms": 1.331, "p95_ms": 1.219}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.342, "count": 204, "max_ms": 86.955, "p95_ms": 77.027}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.707, "count": 204, "max_ms": 120.394, "p95_ms": 107.233}, "yolo11_640_fire_11class": {"avg_ms": 59.858, "count": 204, "max_ms": 89.447, "p95_ms": 80.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.103, "count": 204, "max_ms": 102.857, "p95_ms": 97.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.175, "count": 204, "max_ms": 202.83, "p95_ms": 180.57}, "read_interval": {"avg_ms": 147.265, "count": 204, "max_ms": 211.177, "p95_ms": 188.912}, "triton_wait": {"avg_ms": 75.049, "count": 204, "max_ms": 120.407, "p95_ms": 107.246}, "visualize": {"avg_ms": 5.655, "count": 204, "max_ms": 8.498, "p95_ms": 7.102}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1499}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5933.273, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:06:13+09:00", "last_error_summary": "[h264 @ 0x55db3d26ae00] error while decoding MB 35 16, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 288, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:06:13+09:00", "last_error_summary": "[h264 @ 0x55db3d26ae00] error while decoding MB 35 16, bytestream -5", "last_exit_at": "2026-07-06T19:06:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 177.77, "last_started_at": "2026-07-06T19:03:15+09:00", "pid": 23084, "process": {"exists": false, "pid": 23084}, "restart_count": 84, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.253, "sample_count": 2067, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 179991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:06:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T19:06:13+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-06T19:06:13+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T19:06:13.576498+09:00", "mtime_epoch": 1783332373.5764978, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158239}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:06:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.141, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.823, "pushed": 6.823, "read": 6.789}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.727, "count": 204, "max_ms": 10.823, "p95_ms": 8.102}, "gstreamer_push": {"avg_ms": 1.071, "count": 204, "max_ms": 1.331, "p95_ms": 1.219}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.287, "count": 204, "max_ms": 86.955, "p95_ms": 77.027}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.621, "count": 204, "max_ms": 120.394, "p95_ms": 107.233}, "yolo11_640_fire_11class": {"avg_ms": 59.852, "count": 204, "max_ms": 89.447, "p95_ms": 80.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.093, "count": 204, "max_ms": 102.857, "p95_ms": 97.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.093, "count": 204, "max_ms": 202.83, "p95_ms": 180.57}, "read_interval": {"avg_ms": 147.289, "count": 203, "max_ms": 211.177, "p95_ms": 188.912}, "triton_wait": {"avg_ms": 74.967, "count": 204, "max_ms": 120.407, "p95_ms": 107.246}, "visualize": {"avg_ms": 5.656, "count": 204, "max_ms": 8.498, "p95_ms": 7.102}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1499}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5933.383, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 288, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:06:13+09:00", "last_error_summary": "[h264 @ 0x55db3d26ae00] error while decoding MB 35 16, bytestream -5", "last_exit_at": "2026-07-06T19:06:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 177.775, "last_started_at": "2026-07-06T19:03:15+09:00", "pid": 23084, "process": {"exists": false, "pid": 23084}, "restart_count": 84, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.253, "sample_count": 2067, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 179991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:06:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T19:06:13+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T19:06:13+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T19:06:13.576498+09:00", "mtime_epoch": 1783332373.5764978, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158239}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:06:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.141, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.821, "pushed": 6.821, "read": 6.788}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.727, "count": 204, "max_ms": 10.823, "p95_ms": 8.102}, "gstreamer_push": {"avg_ms": 1.071, "count": 204, "max_ms": 1.331, "p95_ms": 1.219}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.287, "count": 204, "max_ms": 86.955, "p95_ms": 77.027}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.621, "count": 204, "max_ms": 120.394, "p95_ms": 107.233}, "yolo11_640_fire_11class": {"avg_ms": 59.852, "count": 204, "max_ms": 89.447, "p95_ms": 80.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.093, "count": 204, "max_ms": 102.857, "p95_ms": 97.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.093, "count": 204, "max_ms": 202.83, "p95_ms": 180.57}, "read_interval": {"avg_ms": 147.289, "count": 203, "max_ms": 211.177, "p95_ms": 188.912}, "triton_wait": {"avg_ms": 74.967, "count": 204, "max_ms": 120.407, "p95_ms": 107.246}, "visualize": {"avg_ms": 5.656, "count": 204, "max_ms": 8.498, "p95_ms": 7.102}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1499}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6033.566, "limit_mb": 8796093022207.996}, "detail": {"pid": 23191, "restart_count": 85, "start_count": 86}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 288, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:06:13+09:00", "last_error_summary": "[h264 @ 0x55db3d26ae00] error while decoding MB 35 16, bytestream -5", "last_exit_at": "2026-07-06T19:06:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:06:16+09:00", "pid": 23191, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.344, "name": "ffmpeg", "pid": 23191, "rss_mb": 136.344, "state": "S (sleeping)", "threads": 106}, "restart_count": 85, "restart_count_per_hour": 20, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.253, "sample_count": 2034, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 179991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:06:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.62, "last_frame_pushed_at": "2026-07-06T19:06:13+09:00", "last_frame_read_age_sec": 4.736, "last_frame_read_at": "2026-07-06T19:06:13+09:00", "mjpeg": {"age_sec": 4.607, "exists": true, "mtime": "2026-07-06T19:06:13.696500+09:00", "mtime_epoch": 1783332373.6964998, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:06:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.141, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 173, "pushed_frames": 173, "read_frames": 172, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.777, "pushed": 5.777, "read": 5.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.778, "count": 173, "max_ms": 10.823, "p95_ms": 8.449}, "gstreamer_push": {"avg_ms": 1.068, "count": 173, "max_ms": 1.331, "p95_ms": 1.221}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.877, "count": 173, "max_ms": 83.864, "p95_ms": 77.027}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.756, "count": 173, "max_ms": 119.399, "p95_ms": 102.318}, "yolo11_640_fire_11class": {"avg_ms": 59.562, "count": 173, "max_ms": 85.22, "p95_ms": 80.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.688, "count": 173, "max_ms": 102.002, "p95_ms": 97.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.048, "count": 173, "max_ms": 198.188, "p95_ms": 175.713}, "read_interval": {"avg_ms": 147.263, "count": 172, "max_ms": 206.802, "p95_ms": 184.289}, "triton_wait": {"avg_ms": 74.143, "count": 173, "max_ms": 119.41, "p95_ms": 102.331}, "visualize": {"avg_ms": 5.691, "count": 173, "max_ms": 8.498, "p95_ms": 7.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5942.508, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557110e3f140] error while decoding MB 18 42, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 289, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:08:59+09:00", "last_error_summary": "[h264 @ 0x557110e3f140] error while decoding MB 18 42, bytestream -5", "last_exit_at": "2026-07-06T19:06:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.503, "last_started_at": "2026-07-06T19:06:16+09:00", "pid": 23191, "process": {"exists": false, "pid": 23191}, "restart_count": 85, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.62, "sample_count": 2047, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 181109, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:08:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T19:08:59+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T19:08:59+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-06T19:08:59.759517+09:00", "mtime_epoch": 1783332539.7595172, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161497}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:08:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.398, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.176, "pushed": 7.176, "read": 7.176}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.384, "count": 214, "max_ms": 10.839, "p95_ms": 7.705}, "gstreamer_push": {"avg_ms": 1.089, "count": 215, "max_ms": 1.362, "p95_ms": 1.224}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.462, "count": 214, "max_ms": 86.47, "p95_ms": 72.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.534, "count": 214, "max_ms": 116.219, "p95_ms": 99.745}, "yolo11_640_fire_11class": {"avg_ms": 59.846, "count": 214, "max_ms": 90.144, "p95_ms": 83.892}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.374, "count": 214, "max_ms": 104.623, "p95_ms": 97.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.176, "count": 215, "max_ms": 203.611, "p95_ms": 160.014}, "read_interval": {"avg_ms": 139.623, "count": 215, "max_ms": 212.348, "p95_ms": 170.17}, "triton_wait": {"avg_ms": 73.513, "count": 214, "max_ms": 116.232, "p95_ms": 100.877}, "visualize": {"avg_ms": 5.291, "count": 214, "max_ms": 8.142, "p95_ms": 6.388}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5941.281, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:08:59+09:00", "last_error_summary": "[h264 @ 0x557110e3f140] error while decoding MB 18 42, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 289, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:08:59+09:00", "last_error_summary": "[h264 @ 0x557110e3f140] error while decoding MB 18 42, bytestream -5", "last_exit_at": "2026-07-06T19:08:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.579, "last_started_at": "2026-07-06T19:06:16+09:00", "pid": 23191, "process": {"exists": false, "pid": 23191}, "restart_count": 85, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.62, "sample_count": 2048, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 181110, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:08:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T19:08:59+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T19:08:59+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-06T19:08:59.759517+09:00", "mtime_epoch": 1783332539.7595172, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161497}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:08:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.398, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.191, "pushed": 7.191, "read": 7.157}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.383, "count": 215, "max_ms": 10.839, "p95_ms": 7.705}, "gstreamer_push": {"avg_ms": 1.089, "count": 215, "max_ms": 1.362, "p95_ms": 1.224}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.522, "count": 215, "max_ms": 86.47, "p95_ms": 72.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.532, "count": 215, "max_ms": 116.219, "p95_ms": 99.745}, "yolo11_640_fire_11class": {"avg_ms": 59.859, "count": 215, "max_ms": 90.144, "p95_ms": 83.892}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.384, "count": 215, "max_ms": 104.623, "p95_ms": 97.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.216, "count": 215, "max_ms": 203.611, "p95_ms": 160.014}, "read_interval": {"avg_ms": 139.693, "count": 214, "max_ms": 212.348, "p95_ms": 170.17}, "triton_wait": {"avg_ms": 73.507, "count": 215, "max_ms": 116.232, "p95_ms": 100.877}, "visualize": {"avg_ms": 5.291, "count": 215, "max_ms": 8.142, "p95_ms": 6.388}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5941.355, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 289, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:08:59+09:00", "last_error_summary": "[h264 @ 0x557110e3f140] error while decoding MB 18 42, bytestream -5", "last_exit_at": "2026-07-06T19:08:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.584, "last_started_at": "2026-07-06T19:06:16+09:00", "pid": 23191, "process": {"exists": false, "pid": 23191}, "restart_count": 85, "restart_count_per_hour": 19, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.62, "sample_count": 2048, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 181110, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:08:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T19:08:59+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T19:08:59+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-06T19:08:59.759517+09:00", "mtime_epoch": 1783332539.7595172, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161497}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:08:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.398, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.19, "pushed": 7.19, "read": 7.156}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.383, "count": 215, "max_ms": 10.839, "p95_ms": 7.705}, "gstreamer_push": {"avg_ms": 1.089, "count": 215, "max_ms": 1.362, "p95_ms": 1.224}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.522, "count": 215, "max_ms": 86.47, "p95_ms": 72.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.532, "count": 215, "max_ms": 116.219, "p95_ms": 99.745}, "yolo11_640_fire_11class": {"avg_ms": 59.859, "count": 215, "max_ms": 90.144, "p95_ms": 83.892}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.384, "count": 215, "max_ms": 104.623, "p95_ms": 97.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.216, "count": 215, "max_ms": 203.611, "p95_ms": 160.014}, "read_interval": {"avg_ms": 139.693, "count": 214, "max_ms": 212.348, "p95_ms": 170.17}, "triton_wait": {"avg_ms": 73.507, "count": 215, "max_ms": 116.232, "p95_ms": 100.877}, "visualize": {"avg_ms": 5.291, "count": 215, "max_ms": 8.142, "p95_ms": 6.388}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6052.074, "limit_mb": 8796093022207.996}, "detail": {"pid": 23298, "restart_count": 86, "start_count": 87}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 289, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:08:59+09:00", "last_error_summary": "[h264 @ 0x557110e3f140] error while decoding MB 18 42, bytestream -5", "last_exit_at": "2026-07-06T19:09:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:09:02+09:00", "pid": 23298, "process": {"exists": true, "fd_count": 5, "hwm_mb": 150.008, "name": "ffmpeg", "pid": 23298, "rss_mb": 150.008, "state": "S (sleeping)", "threads": 106}, "restart_count": 86, "restart_count_per_hour": 20, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.62, "sample_count": 2017, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 181110, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:08:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.205, "last_frame_pushed_at": "2026-07-06T19:08:59+09:00", "last_frame_read_age_sec": 4.331, "last_frame_read_at": "2026-07-06T19:08:59+09:00", "mjpeg": {"age_sec": 4.186, "exists": true, "mtime": "2026-07-06T19:08:59.895519+09:00", "mtime_epoch": 1783332539.8955195, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161969}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:09:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.398, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.173, "pushed": 6.173, "read": 6.14}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.412, "count": 184, "max_ms": 10.839, "p95_ms": 7.705}, "gstreamer_push": {"avg_ms": 1.089, "count": 185, "max_ms": 1.362, "p95_ms": 1.224}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.936, "count": 184, "max_ms": 86.47, "p95_ms": 71.941}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.467, "count": 184, "max_ms": 116.219, "p95_ms": 98.786}, "yolo11_640_fire_11class": {"avg_ms": 59.425, "count": 184, "max_ms": 90.144, "p95_ms": 83.131}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.184, "count": 184, "max_ms": 104.623, "p95_ms": 96.272}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.798, "count": 185, "max_ms": 203.611, "p95_ms": 160.014}, "read_interval": {"avg_ms": 140.276, "count": 184, "max_ms": 212.348, "p95_ms": 170.478}, "triton_wait": {"avg_ms": 73.444, "count": 184, "max_ms": 116.232, "p95_ms": 100.87}, "visualize": {"avg_ms": 5.306, "count": 184, "max_ms": 8.142, "p95_ms": 6.368}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5943.078, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55676c342640] reference picture missing during reorder [h264 @ 0x55676c342640] Missing reference picture, default is 65576 [h264 @ 0x55676c356800] mmco: unref short failure [h264 @ 0x55676c356800] illegal short term buffer state detected [h264 @ 0x55676c342640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c342640] error while decoding MB 0 35, bytestream 1196 [h264 @ 0x55676c356800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c356800] er", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:11:50+09:00", "last_error_summary": "[h264 @ 0x55676c342640] reference picture missing during reorder [h264 @ 0x55676c342640] Missing reference picture, default is 65576 [h264 @ 0x55676c356800] mmco: unref short failure [h264 @ 0x55676c356800] illegal short term buffer state detected [h264 @ 0x55676c342640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c342640] error while decoding MB 0 35, bytestream 1196 [h264 @ 0x55676c356800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c356800] er", "last_exit_at": "2026-07-06T19:09:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 168.867, "last_started_at": "2026-07-06T19:09:02+09:00", "pid": 23298, "process": {"exists": false, "pid": 23298}, "restart_count": 86, "restart_count_per_hour": 19, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.493, "sample_count": 1961, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182180, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:11:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-06T19:11:50+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T19:11:50+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-06T19:11:50.898629+09:00", "mtime_epoch": 1783332710.8986294, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:11:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 84, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.802, "pushed": 2.802, "read": 2.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.642, "count": 83, "max_ms": 8.697, "p95_ms": 7.037}, "gstreamer_push": {"avg_ms": 1.171, "count": 84, "max_ms": 2.059, "p95_ms": 1.37}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.221, "count": 83, "max_ms": 84.898, "p95_ms": 75.118}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.64, "count": 83, "max_ms": 108.647, "p95_ms": 98.995}, "yolo11_640_fire_11class": {"avg_ms": 61.557, "count": 83, "max_ms": 88.878, "p95_ms": 83.336}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.849, "count": 83, "max_ms": 100.839, "p95_ms": 92.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.266, "count": 84, "max_ms": 187.973, "p95_ms": 159.608}, "read_interval": {"avg_ms": 357.619, "count": 84, "max_ms": 19281.448, "p95_ms": 175.184}, "triton_wait": {"avg_ms": 73.457, "count": 83, "max_ms": 108.657, "p95_ms": 100.023}, "visualize": {"avg_ms": 4.994, "count": 83, "max_ms": 8.436, "p95_ms": 6.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5942.648, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:11:50+09:00", "last_error_summary": "[h264 @ 0x55676c342640] reference picture missing during reorder [h264 @ 0x55676c342640] Missing reference picture, default is 65576 [h264 @ 0x55676c356800] mmco: unref short failure [h264 @ 0x55676c356800] illegal short term buffer state detected [h264 @ 0x55676c342640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c342640] error while decoding MB 0 35, bytestream 1196 [h264 @ 0x55676c356800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c356800] er", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:11:50+09:00", "last_error_summary": "[h264 @ 0x55676c342640] reference picture missing during reorder [h264 @ 0x55676c342640] Missing reference picture, default is 65576 [h264 @ 0x55676c356800] mmco: unref short failure [h264 @ 0x55676c356800] illegal short term buffer state detected [h264 @ 0x55676c342640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c342640] error while decoding MB 0 35, bytestream 1196 [h264 @ 0x55676c356800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c356800] er", "last_exit_at": "2026-07-06T19:11:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 168.941, "last_started_at": "2026-07-06T19:09:02+09:00", "pid": 23298, "process": {"exists": false, "pid": 23298}, "restart_count": 86, "restart_count_per_hour": 19, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.493, "sample_count": 1961, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:11:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T19:11:51+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T19:11:50+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T19:11:50.898629+09:00", "mtime_epoch": 1783332710.8986294, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:11:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.914, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.806, "pushed": 2.806, "read": 2.773}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.655, "count": 84, "max_ms": 8.697, "p95_ms": 7.037}, "gstreamer_push": {"avg_ms": 1.171, "count": 84, "max_ms": 2.059, "p95_ms": 1.37}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.226, "count": 84, "max_ms": 84.898, "p95_ms": 75.118}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.59, "count": 84, "max_ms": 108.647, "p95_ms": 98.995}, "yolo11_640_fire_11class": {"avg_ms": 61.578, "count": 84, "max_ms": 88.878, "p95_ms": 83.336}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.834, "count": 84, "max_ms": 100.839, "p95_ms": 92.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.395, "count": 84, "max_ms": 187.973, "p95_ms": 159.608}, "read_interval": {"avg_ms": 360.477, "count": 83, "max_ms": 19281.448, "p95_ms": 175.184}, "triton_wait": {"avg_ms": 73.397, "count": 84, "max_ms": 108.657, "p95_ms": 100.023}, "visualize": {"avg_ms": 4.994, "count": 84, "max_ms": 8.436, "p95_ms": 6.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5942.641, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:11:50+09:00", "last_error_summary": "[h264 @ 0x55676c342640] reference picture missing during reorder [h264 @ 0x55676c342640] Missing reference picture, default is 65576 [h264 @ 0x55676c356800] mmco: unref short failure [h264 @ 0x55676c356800] illegal short term buffer state detected [h264 @ 0x55676c342640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c342640] error while decoding MB 0 35, bytestream 1196 [h264 @ 0x55676c356800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c356800] er", "last_exit_at": "2026-07-06T19:11:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 168.946, "last_started_at": "2026-07-06T19:09:02+09:00", "pid": 23298, "process": {"exists": false, "pid": 23298}, "restart_count": 86, "restart_count_per_hour": 19, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.493, "sample_count": 1961, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:11:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T19:11:51+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T19:11:50+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-06T19:11:50.898629+09:00", "mtime_epoch": 1783332710.8986294, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:11:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.914, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.806, "pushed": 2.806, "read": 2.772}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.655, "count": 84, "max_ms": 8.697, "p95_ms": 7.037}, "gstreamer_push": {"avg_ms": 1.171, "count": 84, "max_ms": 2.059, "p95_ms": 1.37}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.226, "count": 84, "max_ms": 84.898, "p95_ms": 75.118}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.59, "count": 84, "max_ms": 108.647, "p95_ms": 98.995}, "yolo11_640_fire_11class": {"avg_ms": 61.578, "count": 84, "max_ms": 88.878, "p95_ms": 83.336}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.834, "count": 84, "max_ms": 100.839, "p95_ms": 92.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.395, "count": 84, "max_ms": 187.973, "p95_ms": 159.608}, "read_interval": {"avg_ms": 360.477, "count": 83, "max_ms": 19281.448, "p95_ms": 175.184}, "triton_wait": {"avg_ms": 73.397, "count": 84, "max_ms": 108.657, "p95_ms": 100.023}, "visualize": {"avg_ms": 4.994, "count": 84, "max_ms": 8.436, "p95_ms": 6.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5954.539, "limit_mb": 8796093022207.996}, "detail": {"pid": 23405, "restart_count": 87, "start_count": 88}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:11:50+09:00", "last_error_summary": "[h264 @ 0x55676c342640] reference picture missing during reorder [h264 @ 0x55676c342640] Missing reference picture, default is 65576 [h264 @ 0x55676c356800] mmco: unref short failure [h264 @ 0x55676c356800] illegal short term buffer state detected [h264 @ 0x55676c342640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c342640] error while decoding MB 0 35, bytestream 1196 [h264 @ 0x55676c356800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c356800] er", "last_exit_at": "2026-07-06T19:11:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:11:53+09:00", "pid": 23405, "process": {"exists": true, "fd_count": 4, "hwm_mb": 46.457, "name": "ffmpeg", "pid": 23405, "rss_mb": 46.457, "state": "S (sleeping)", "threads": 1}, "restart_count": 87, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.536, "sample_count": 1929, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:11:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.689, "last_frame_pushed_at": "2026-07-06T19:11:51+09:00", "last_frame_read_age_sec": 4.813, "last_frame_read_at": "2026-07-06T19:11:50+09:00", "mjpeg": {"age_sec": 4.671, "exists": true, "mtime": "2026-07-06T19:11:51.034632+09:00", "mtime_epoch": 1783332711.034632, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:11:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.671, "pushed": 1.671, "read": 1.637}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.346, "count": 50, "max_ms": 8.697, "p95_ms": 7.037}, "gstreamer_push": {"avg_ms": 1.204, "count": 50, "max_ms": 2.059, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.398, "count": 50, "max_ms": 84.898, "p95_ms": 77.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.117, "count": 50, "max_ms": 108.647, "p95_ms": 100.172}, "yolo11_640_fire_11class": {"avg_ms": 61.891, "count": 50, "max_ms": 88.878, "p95_ms": 83.457}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.556, "count": 50, "max_ms": 100.014, "p95_ms": 94.705}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.12, "count": 50, "max_ms": 187.973, "p95_ms": 167.219}, "read_interval": {"avg_ms": 515.587, "count": 49, "max_ms": 19281.448, "p95_ms": 177.405}, "triton_wait": {"avg_ms": 73.491, "count": 50, "max_ms": 108.657, "p95_ms": 100.19}, "visualize": {"avg_ms": 4.89, "count": 50, "max_ms": 8.436, "p95_ms": 6.474}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5971.363, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:11:50+09:00", "last_error_summary": "[h264 @ 0x55676c342640] reference picture missing during reorder [h264 @ 0x55676c342640] Missing reference picture, default is 65576 [h264 @ 0x55676c356800] mmco: unref short failure [h264 @ 0x55676c356800] illegal short term buffer state detected [h264 @ 0x55676c342640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c342640] error while decoding MB 0 35, bytestream 1196 [h264 @ 0x55676c356800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c356800] er", "last_exit_at": "2026-07-06T19:11:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 32.059, "last_started_at": "2026-07-06T19:11:53+09:00", "pid": 23405, "process": {"exists": true, "fd_count": 5, "hwm_mb": 64.316, "name": "ffmpeg", "pid": 23405, "rss_mb": 64.316, "state": "S (sleeping)", "threads": 19}, "restart_count": 87, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.871, "sample_count": 1730, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:11:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 34.747, "last_frame_pushed_at": "2026-07-06T19:11:51+09:00", "last_frame_read_age_sec": 34.871, "last_frame_read_at": "2026-07-06T19:11:50+09:00", "mjpeg": {"age_sec": 34.729, "exists": true, "mtime": "2026-07-06T19:11:51.034632+09:00", "mtime_epoch": 1783332711.034632, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:12:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5961.551, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:11:50+09:00", "last_error_summary": "[h264 @ 0x55676c342640] reference picture missing during reorder [h264 @ 0x55676c342640] Missing reference picture, default is 65576 [h264 @ 0x55676c356800] mmco: unref short failure [h264 @ 0x55676c356800] illegal short term buffer state detected [h264 @ 0x55676c342640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c342640] error while decoding MB 0 35, bytestream 1196 [h264 @ 0x55676c356800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55676c356800] er", "last_exit_at": "2026-07-06T19:11:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 32.068, "last_started_at": "2026-07-06T19:11:53+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 87, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.871, "sample_count": 1730, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:11:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 34.755, "last_frame_pushed_at": "2026-07-06T19:11:51+09:00", "last_frame_read_age_sec": 34.88, "last_frame_read_at": "2026-07-06T19:11:50+09:00", "mjpeg": {"age_sec": 34.738, "exists": true, "mtime": "2026-07-06T19:11:51.034632+09:00", "mtime_epoch": 1783332711.034632, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:12:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5943.562, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56437932c300] error while decoding MB 45 21, bytestream -26 [h264 @ 0x564379331b80] error while decoding MB 45 21, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 296, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:12:25+09:00", "last_error_summary": "[h264 @ 0x56437932c300] error while decoding MB 45 21, bytestream -26 [h264 @ 0x564379331b80] error while decoding MB 45 21, bytestream -26", "last_exit_at": "2026-07-06T19:11:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 32.076, "last_started_at": "2026-07-06T19:11:53+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 87, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.871, "sample_count": 1730, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:11:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 34.764, "last_frame_pushed_at": "2026-07-06T19:11:51+09:00", "last_frame_read_age_sec": 34.889, "last_frame_read_at": "2026-07-06T19:11:50+09:00", "mjpeg": {"age_sec": 34.747, "exists": true, "mtime": "2026-07-06T19:11:51.034632+09:00", "mtime_epoch": 1783332711.034632, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:12:25+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6052.375, "limit_mb": 8796093022207.996}, "detail": {"pid": 23425, "restart_count": 88, "start_count": 89}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 296, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:12:25+09:00", "last_error_summary": "[h264 @ 0x56437932c300] error while decoding MB 45 21, bytestream -26 [h264 @ 0x564379331b80] error while decoding MB 45 21, bytestream -26", "last_exit_at": "2026-07-06T19:11:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:12:29+09:00", "pid": 23425, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.723, "name": "ffmpeg", "pid": 23425, "rss_mb": 141.723, "state": "S (sleeping)", "threads": 106}, "restart_count": 88, "restart_count_per_hour": 21, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.904, "sample_count": 1690, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:11:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 40.822, "last_frame_pushed_at": "2026-07-06T19:11:51+09:00", "last_frame_read_age_sec": 40.946, "last_frame_read_at": "2026-07-06T19:11:50+09:00", "mjpeg": {"age_sec": 40.804, "exists": true, "mtime": "2026-07-06T19:11:51.034632+09:00", "mtime_epoch": 1783332711.034632, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:12:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1624.914, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5935.586, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d5a0448c40] error while decoding MB 9 20, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 297, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:15:13+09:00", "last_error_summary": "[h264 @ 0x55d5a0448c40] error while decoding MB 9 20, bytestream -12", "last_exit_at": "2026-07-06T19:11:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.604, "last_started_at": "2026-07-06T19:12:29+09:00", "pid": 23425, "process": {"exists": false, "pid": 23425}, "restart_count": 88, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.825, "sample_count": 1623, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 183240, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:15:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.044, "last_frame_pushed_at": "2026-07-06T19:15:13+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-06T19:15:13+09:00", "mjpeg": {"age_sec": 0.027, "exists": true, "mtime": "2026-07-06T19:15:13.414462+09:00", "mtime_epoch": 1783332913.4144619, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154857}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:15:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.43, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 183, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.118, "pushed": 6.118, "read": 6.118}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.981, "count": 183, "max_ms": 10.867, "p95_ms": 8.887}, "gstreamer_push": {"avg_ms": 1.074, "count": 183, "max_ms": 1.368, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.067, "count": 183, "max_ms": 84.636, "p95_ms": 71.221}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.485, "count": 183, "max_ms": 121.802, "p95_ms": 100.213}, "yolo11_640_fire_11class": {"avg_ms": 61.064, "count": 183, "max_ms": 93.127, "p95_ms": 81.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.124, "count": 183, "max_ms": 105.358, "p95_ms": 94.456}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.525, "count": 183, "max_ms": 203.699, "p95_ms": 178.289}, "read_interval": {"avg_ms": 164.011, "count": 183, "max_ms": 1912.697, "p95_ms": 187.705}, "triton_wait": {"avg_ms": 75.172, "count": 183, "max_ms": 121.815, "p95_ms": 100.226}, "visualize": {"avg_ms": 6.02, "count": 183, "max_ms": 8.327, "p95_ms": 7.337}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5936.773, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:15:13+09:00", "last_error_summary": "[h264 @ 0x55d5a0448c40] error while decoding MB 9 20, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 297, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:15:13+09:00", "last_error_summary": "[h264 @ 0x55d5a0448c40] error while decoding MB 9 20, bytestream -12", "last_exit_at": "2026-07-06T19:15:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.718, "last_started_at": "2026-07-06T19:12:29+09:00", "pid": 23425, "process": {"exists": false, "pid": 23425}, "restart_count": 88, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.825, "sample_count": 1623, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 183241, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:15:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T19:15:13+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-06T19:15:13+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T19:15:13.414462+09:00", "mtime_epoch": 1783332913.4144619, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154857}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:15:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.43, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.121, "pushed": 6.121, "read": 6.087}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.984, "count": 183, "max_ms": 10.867, "p95_ms": 8.887}, "gstreamer_push": {"avg_ms": 1.073, "count": 183, "max_ms": 1.368, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.012, "count": 183, "max_ms": 84.636, "p95_ms": 71.221}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.358, "count": 183, "max_ms": 121.802, "p95_ms": 100.213}, "yolo11_640_fire_11class": {"avg_ms": 61.067, "count": 183, "max_ms": 93.127, "p95_ms": 81.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.139, "count": 183, "max_ms": 105.358, "p95_ms": 94.456}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.536, "count": 183, "max_ms": 203.699, "p95_ms": 178.289}, "read_interval": {"avg_ms": 164.084, "count": 182, "max_ms": 1912.697, "p95_ms": 187.705}, "triton_wait": {"avg_ms": 75.044, "count": 183, "max_ms": 121.815, "p95_ms": 100.226}, "visualize": {"avg_ms": 6.025, "count": 183, "max_ms": 8.327, "p95_ms": 7.337}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5936.766, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 297, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:15:13+09:00", "last_error_summary": "[h264 @ 0x55d5a0448c40] error while decoding MB 9 20, bytestream -12", "last_exit_at": "2026-07-06T19:15:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.723, "last_started_at": "2026-07-06T19:12:29+09:00", "pid": 23425, "process": {"exists": false, "pid": 23425}, "restart_count": 88, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.825, "sample_count": 1623, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 183241, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:15:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T19:15:13+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-06T19:15:13+09:00", "mjpeg": {"age_sec": 0.146, "exists": true, "mtime": "2026-07-06T19:15:13.414462+09:00", "mtime_epoch": 1783332913.4144619, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154857}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:15:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.43, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.12, "pushed": 6.12, "read": 6.086}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.984, "count": 183, "max_ms": 10.867, "p95_ms": 8.887}, "gstreamer_push": {"avg_ms": 1.073, "count": 183, "max_ms": 1.368, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.012, "count": 183, "max_ms": 84.636, "p95_ms": 71.221}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.358, "count": 183, "max_ms": 121.802, "p95_ms": 100.213}, "yolo11_640_fire_11class": {"avg_ms": 61.067, "count": 183, "max_ms": 93.127, "p95_ms": 81.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.139, "count": 183, "max_ms": 105.358, "p95_ms": 94.456}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.536, "count": 183, "max_ms": 203.699, "p95_ms": 178.289}, "read_interval": {"avg_ms": 164.084, "count": 182, "max_ms": 1912.697, "p95_ms": 187.705}, "triton_wait": {"avg_ms": 75.044, "count": 183, "max_ms": 121.815, "p95_ms": 100.226}, "visualize": {"avg_ms": 6.025, "count": 183, "max_ms": 8.327, "p95_ms": 7.337}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6048.91, "limit_mb": 8796093022207.996}, "detail": {"pid": 23532, "restart_count": 89, "start_count": 90}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 297, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:15:13+09:00", "last_error_summary": "[h264 @ 0x55d5a0448c40] error while decoding MB 9 20, bytestream -12", "last_exit_at": "2026-07-06T19:15:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:15:15+09:00", "pid": 23532, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.082, "name": "ffmpeg", "pid": 23532, "rss_mb": 144.082, "state": "S (sleeping)", "threads": 106}, "restart_count": 89, "restart_count_per_hour": 21, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.825, "sample_count": 1592, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 183241, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:15:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.369, "last_frame_pushed_at": "2026-07-06T19:15:13+09:00", "last_frame_read_age_sec": 4.514, "last_frame_read_at": "2026-07-06T19:15:13+09:00", "mjpeg": {"age_sec": 4.352, "exists": true, "mtime": "2026-07-06T19:15:13.566466+09:00", "mtime_epoch": 1783332913.5664663, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151604}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:15:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.43, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 157, "pushed_frames": 157, "read_frames": 156, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.258, "pushed": 5.258, "read": 5.224}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.976, "count": 157, "max_ms": 10.867, "p95_ms": 9.078}, "gstreamer_push": {"avg_ms": 1.074, "count": 157, "max_ms": 1.368, "p95_ms": 1.237}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.918, "count": 157, "max_ms": 84.636, "p95_ms": 71.293}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.269, "count": 157, "max_ms": 121.802, "p95_ms": 99.8}, "yolo11_640_fire_11class": {"avg_ms": 60.849, "count": 157, "max_ms": 93.127, "p95_ms": 81.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.008, "count": 157, "max_ms": 105.358, "p95_ms": 95.768}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.155, "count": 157, "max_ms": 203.699, "p95_ms": 177.52}, "read_interval": {"avg_ms": 163.424, "count": 156, "max_ms": 1912.697, "p95_ms": 186.538}, "triton_wait": {"avg_ms": 73.944, "count": 157, "max_ms": 121.815, "p95_ms": 99.835}, "visualize": {"avg_ms": 5.973, "count": 157, "max_ms": 8.327, "p95_ms": 7.465}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5948.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55957ed288c0] error while decoding MB 69 36, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 298, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:17:59+09:00", "last_error_summary": "[h264 @ 0x55957ed288c0] error while decoding MB 69 36, bytestream -5", "last_exit_at": "2026-07-06T19:15:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.549, "last_started_at": "2026-07-06T19:15:15+09:00", "pid": 23532, "process": {"exists": false, "pid": 23532}, "restart_count": 89, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.637, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 184339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:17:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T19:17:59+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T19:17:59+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T19:17:59.427049+09:00", "mtime_epoch": 1783333079.4270494, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:17:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.945, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.818, "pushed": 6.818, "read": 6.818}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.614, "count": 204, "max_ms": 11.014, "p95_ms": 7.698}, "gstreamer_push": {"avg_ms": 1.059, "count": 204, "max_ms": 1.307, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.659, "count": 204, "max_ms": 89.224, "p95_ms": 75.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.461, "count": 204, "max_ms": 117.535, "p95_ms": 99.289}, "yolo11_640_fire_11class": {"avg_ms": 60.309, "count": 204, "max_ms": 89.706, "p95_ms": 80.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.444, "count": 204, "max_ms": 102.156, "p95_ms": 93.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.094, "count": 204, "max_ms": 197.19, "p95_ms": 175.58}, "read_interval": {"avg_ms": 147.105, "count": 204, "max_ms": 205.846, "p95_ms": 185.972}, "triton_wait": {"avg_ms": 74.122, "count": 204, "max_ms": 117.547, "p95_ms": 99.943}, "visualize": {"avg_ms": 5.673, "count": 204, "max_ms": 9.925, "p95_ms": 6.865}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5948.062, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:17:59+09:00", "last_error_summary": "[h264 @ 0x55957ed288c0] error while decoding MB 69 36, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 298, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:17:59+09:00", "last_error_summary": "[h264 @ 0x55957ed288c0] error while decoding MB 69 36, bytestream -5", "last_exit_at": "2026-07-06T19:17:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.654, "last_started_at": "2026-07-06T19:15:15+09:00", "pid": 23532, "process": {"exists": false, "pid": 23532}, "restart_count": 89, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.637, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 184340, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:17:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T19:17:59+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-06T19:17:59+09:00", "mjpeg": {"age_sec": 0.144, "exists": true, "mtime": "2026-07-06T19:17:59.427049+09:00", "mtime_epoch": 1783333079.4270494, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:17:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.945, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.823, "pushed": 6.823, "read": 6.79}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.618, "count": 204, "max_ms": 11.014, "p95_ms": 7.698}, "gstreamer_push": {"avg_ms": 1.059, "count": 204, "max_ms": 1.307, "p95_ms": 1.192}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.556, "count": 204, "max_ms": 89.224, "p95_ms": 75.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.483, "count": 204, "max_ms": 117.535, "p95_ms": 99.289}, "yolo11_640_fire_11class": {"avg_ms": 60.242, "count": 204, "max_ms": 89.706, "p95_ms": 80.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.359, "count": 204, "max_ms": 102.156, "p95_ms": 93.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.123, "count": 204, "max_ms": 197.19, "p95_ms": 175.58}, "read_interval": {"avg_ms": 147.094, "count": 203, "max_ms": 205.846, "p95_ms": 185.972}, "triton_wait": {"avg_ms": 74.144, "count": 204, "max_ms": 117.547, "p95_ms": 99.943}, "visualize": {"avg_ms": 5.674, "count": 204, "max_ms": 9.925, "p95_ms": 6.865}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5948.055, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 298, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:17:59+09:00", "last_error_summary": "[h264 @ 0x55957ed288c0] error while decoding MB 69 36, bytestream -5", "last_exit_at": "2026-07-06T19:17:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.659, "last_started_at": "2026-07-06T19:15:15+09:00", "pid": 23532, "process": {"exists": false, "pid": 23532}, "restart_count": 89, "restart_count_per_hour": 20, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.637, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 184340, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:17:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T19:17:59+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-06T19:17:59+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-06T19:17:59.427049+09:00", "mtime_epoch": 1783333079.4270494, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:17:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.945, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.822, "pushed": 6.822, "read": 6.789}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.618, "count": 204, "max_ms": 11.014, "p95_ms": 7.698}, "gstreamer_push": {"avg_ms": 1.059, "count": 204, "max_ms": 1.307, "p95_ms": 1.192}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.556, "count": 204, "max_ms": 89.224, "p95_ms": 75.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.483, "count": 204, "max_ms": 117.535, "p95_ms": 99.289}, "yolo11_640_fire_11class": {"avg_ms": 60.242, "count": 204, "max_ms": 89.706, "p95_ms": 80.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.359, "count": 204, "max_ms": 102.156, "p95_ms": 93.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.123, "count": 204, "max_ms": 197.19, "p95_ms": 175.58}, "read_interval": {"avg_ms": 147.094, "count": 203, "max_ms": 205.846, "p95_ms": 185.972}, "triton_wait": {"avg_ms": 74.144, "count": 204, "max_ms": 117.547, "p95_ms": 99.943}, "visualize": {"avg_ms": 5.674, "count": 204, "max_ms": 9.925, "p95_ms": 6.865}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6055.547, "limit_mb": 8796093022207.996}, "detail": {"pid": 23639, "restart_count": 90, "start_count": 91}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 298, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:17:59+09:00", "last_error_summary": "[h264 @ 0x55957ed288c0] error while decoding MB 69 36, bytestream -5", "last_exit_at": "2026-07-06T19:18:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:18:02+09:00", "pid": 23639, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.258, "name": "ffmpeg", "pid": 23639, "rss_mb": 147.258, "state": "S (sleeping)", "threads": 106}, "restart_count": 90, "restart_count_per_hour": 21, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.399, "sample_count": 1939, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 184340, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:17:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.467, "last_frame_pushed_at": "2026-07-06T19:17:59+09:00", "last_frame_read_age_sec": 4.614, "last_frame_read_at": "2026-07-06T19:17:59+09:00", "mjpeg": {"age_sec": 4.451, "exists": true, "mtime": "2026-07-06T19:17:59.579053+09:00", "mtime_epoch": 1783333079.5790534, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:18:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1625.945, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 173, "pushed_frames": 173, "read_frames": 172, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.781, "pushed": 5.781, "read": 5.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.677, "count": 173, "max_ms": 11.014, "p95_ms": 7.698}, "gstreamer_push": {"avg_ms": 1.057, "count": 173, "max_ms": 1.295, "p95_ms": 1.192}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.285, "count": 173, "max_ms": 89.224, "p95_ms": 75.066}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.997, "count": 173, "max_ms": 117.535, "p95_ms": 97.931}, "yolo11_640_fire_11class": {"avg_ms": 59.877, "count": 173, "max_ms": 89.706, "p95_ms": 80.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.963, "count": 173, "max_ms": 102.156, "p95_ms": 92.992}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.878, "count": 173, "max_ms": 197.19, "p95_ms": 175.664}, "read_interval": {"avg_ms": 147.794, "count": 172, "max_ms": 205.846, "p95_ms": 186.289}, "triton_wait": {"avg_ms": 73.754, "count": 173, "max_ms": 117.547, "p95_ms": 99.911}, "visualize": {"avg_ms": 5.73, "count": 173, "max_ms": 9.925, "p95_ms": 6.876}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1543}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5946.836, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55cbb305da40] cabac decode of qscale diff failed at 53 40 [h264 @ 0x55cbb305da40] error while decoding MB 53 40, bytestream 5661 [h264 @ 0x55cbb2f8ebc0] error while decoding MB 12 10, bytestream -10 [h264 @ 0x55cbb24872c0] error while decoding MB 55 32, bytestream -20 [in#0/rtsp @ 0x55cbb2350e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cbb2350e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 302, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:20:24+09:00", "last_error_summary": "[h264 @ 0x55cbb305da40] cabac decode of qscale diff failed at 53 40 [h264 @ 0x55cbb305da40] error while decoding MB 53 40, bytestream 5661 [h264 @ 0x55cbb2f8ebc0] error while decoding MB 12 10, bytestream -10 [h264 @ 0x55cbb24872c0] error while decoding MB 55 32, bytestream -20 [in#0/rtsp @ 0x55cbb2350e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cbb2350e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T19:18:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 142.567, "last_started_at": "2026-07-06T19:18:02+09:00", "pid": 23639, "process": {"exists": false, "pid": 23639}, "restart_count": 90, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.664, "sample_count": 1893, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 185178, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:20:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-06T19:20:24+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T19:20:24+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-06T19:20:24.550767+09:00", "mtime_epoch": 1783333224.5507667, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168653}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:20:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.461, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 79, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.641, "pushed": 2.641, "read": 2.641}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.319, "count": 79, "max_ms": 8.865, "p95_ms": 7.072}, "gstreamer_push": {"avg_ms": 1.086, "count": 79, "max_ms": 1.326, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.974, "count": 79, "max_ms": 85.686, "p95_ms": 79.2}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.148, "count": 79, "max_ms": 122.862, "p95_ms": 111.806}, "yolo11_640_fire_11class": {"avg_ms": 62.226, "count": 79, "max_ms": 90.258, "p95_ms": 88.941}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.113, "count": 79, "max_ms": 103.294, "p95_ms": 94.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.722, "count": 79, "max_ms": 189.464, "p95_ms": 176.058}, "read_interval": {"avg_ms": 379.3, "count": 79, "max_ms": 10127.549, "p95_ms": 185.934}, "triton_wait": {"avg_ms": 78.423, "count": 79, "max_ms": 122.874, "p95_ms": 111.819}, "visualize": {"avg_ms": 5.311, "count": 79, "max_ms": 7.498, "p95_ms": 6.375}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1543}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5946.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:20:24+09:00", "last_error_summary": "[h264 @ 0x55cbb305da40] cabac decode of qscale diff failed at 53 40 [h264 @ 0x55cbb305da40] error while decoding MB 53 40, bytestream 5661 [h264 @ 0x55cbb2f8ebc0] error while decoding MB 12 10, bytestream -10 [h264 @ 0x55cbb24872c0] error while decoding MB 55 32, bytestream -20 [in#0/rtsp @ 0x55cbb2350e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cbb2350e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 302, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:20:24+09:00", "last_error_summary": "[h264 @ 0x55cbb305da40] cabac decode of qscale diff failed at 53 40 [h264 @ 0x55cbb305da40] error while decoding MB 53 40, bytestream 5661 [h264 @ 0x55cbb2f8ebc0] error while decoding MB 12 10, bytestream -10 [h264 @ 0x55cbb24872c0] error while decoding MB 55 32, bytestream -20 [in#0/rtsp @ 0x55cbb2350e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cbb2350e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T19:20:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 142.63, "last_started_at": "2026-07-06T19:18:02+09:00", "pid": 23639, "process": {"exists": false, "pid": 23639}, "restart_count": 90, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.664, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 185179, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:20:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T19:20:24+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T19:20:24+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T19:20:24.550767+09:00", "mtime_epoch": 1783333224.5507667, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168653}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:20:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.461, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 79, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.669, "pushed": 2.669, "read": 2.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.371, "count": 79, "max_ms": 10.418, "p95_ms": 7.107}, "gstreamer_push": {"avg_ms": 1.085, "count": 80, "max_ms": 1.326, "p95_ms": 1.22}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.994, "count": 79, "max_ms": 85.686, "p95_ms": 79.2}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.126, "count": 79, "max_ms": 122.862, "p95_ms": 111.806}, "yolo11_640_fire_11class": {"avg_ms": 62.151, "count": 79, "max_ms": 90.258, "p95_ms": 88.941}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.059, "count": 79, "max_ms": 103.294, "p95_ms": 94.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.518, "count": 80, "max_ms": 189.464, "p95_ms": 170.791}, "read_interval": {"avg_ms": 379.3, "count": 79, "max_ms": 10127.549, "p95_ms": 185.934}, "triton_wait": {"avg_ms": 78.401, "count": 79, "max_ms": 122.874, "p95_ms": 111.819}, "visualize": {"avg_ms": 5.339, "count": 79, "max_ms": 7.498, "p95_ms": 6.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1543}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5946.363, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 302, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:20:24+09:00", "last_error_summary": "[h264 @ 0x55cbb305da40] cabac decode of qscale diff failed at 53 40 [h264 @ 0x55cbb305da40] error while decoding MB 53 40, bytestream 5661 [h264 @ 0x55cbb2f8ebc0] error while decoding MB 12 10, bytestream -10 [h264 @ 0x55cbb24872c0] error while decoding MB 55 32, bytestream -20 [in#0/rtsp @ 0x55cbb2350e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cbb2350e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T19:20:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 142.635, "last_started_at": "2026-07-06T19:18:02+09:00", "pid": 23639, "process": {"exists": false, "pid": 23639}, "restart_count": 90, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.664, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 185179, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:20:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T19:20:24+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-06T19:20:24+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T19:20:24.550767+09:00", "mtime_epoch": 1783333224.5507667, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168653}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:20:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.461, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 79, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.668, "pushed": 2.668, "read": 2.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.371, "count": 79, "max_ms": 10.418, "p95_ms": 7.107}, "gstreamer_push": {"avg_ms": 1.085, "count": 80, "max_ms": 1.326, "p95_ms": 1.22}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.994, "count": 79, "max_ms": 85.686, "p95_ms": 79.2}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.126, "count": 79, "max_ms": 122.862, "p95_ms": 111.806}, "yolo11_640_fire_11class": {"avg_ms": 62.151, "count": 79, "max_ms": 90.258, "p95_ms": 88.941}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.059, "count": 79, "max_ms": 103.294, "p95_ms": 94.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.518, "count": 80, "max_ms": 189.464, "p95_ms": 170.791}, "read_interval": {"avg_ms": 379.3, "count": 79, "max_ms": 10127.549, "p95_ms": 185.934}, "triton_wait": {"avg_ms": 78.401, "count": 79, "max_ms": 122.874, "p95_ms": 111.819}, "visualize": {"avg_ms": 5.339, "count": 79, "max_ms": 7.498, "p95_ms": 6.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1543}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6059.254, "limit_mb": 8796093022207.996}, "detail": {"pid": 23746, "restart_count": 91, "start_count": 92}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 302, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:20:24+09:00", "last_error_summary": "[h264 @ 0x55cbb305da40] cabac decode of qscale diff failed at 53 40 [h264 @ 0x55cbb305da40] error while decoding MB 53 40, bytestream 5661 [h264 @ 0x55cbb2f8ebc0] error while decoding MB 12 10, bytestream -10 [h264 @ 0x55cbb24872c0] error while decoding MB 55 32, bytestream -20 [in#0/rtsp @ 0x55cbb2350e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cbb2350e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T19:20:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:20:27+09:00", "pid": 23746, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.637, "name": "ffmpeg", "pid": 23746, "rss_mb": 141.637, "state": "S (sleeping)", "threads": 106}, "restart_count": 91, "restart_count_per_hour": 22, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.007, "sample_count": 1861, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 185179, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:20:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.404, "last_frame_pushed_at": "2026-07-06T19:20:24+09:00", "last_frame_read_age_sec": 4.525, "last_frame_read_at": "2026-07-06T19:20:24+09:00", "mjpeg": {"age_sec": 4.389, "exists": true, "mtime": "2026-07-06T19:20:24.670770+09:00", "mtime_epoch": 1783333224.6707697, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168749}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:20:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.461, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.671, "pushed": 1.671, "read": 1.637}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.255, "count": 50, "max_ms": 10.418, "p95_ms": 6.864}, "gstreamer_push": {"avg_ms": 1.099, "count": 50, "max_ms": 1.326, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.875, "count": 50, "max_ms": 85.686, "p95_ms": 78.416}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.05, "count": 50, "max_ms": 111.806, "p95_ms": 104.074}, "yolo11_640_fire_11class": {"avg_ms": 61.905, "count": 50, "max_ms": 90.258, "p95_ms": 82.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.898, "count": 50, "max_ms": 103.294, "p95_ms": 94.079}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.722, "count": 50, "max_ms": 189.464, "p95_ms": 161.343}, "read_interval": {"avg_ms": 521.113, "count": 49, "max_ms": 10127.549, "p95_ms": 199.295}, "triton_wait": {"avg_ms": 78.203, "count": 50, "max_ms": 111.819, "p95_ms": 104.086}, "visualize": {"avg_ms": 5.265, "count": 50, "max_ms": 7.077, "p95_ms": 6.254}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5947.762, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ef94605840] error while decoding MB 19 14, bytestream -18", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 303, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:23:06+09:00", "last_error_summary": "[h264 @ 0x55ef94605840] error while decoding MB 19 14, bytestream -18", "last_exit_at": "2026-07-06T19:20:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.732, "last_started_at": "2026-07-06T19:20:27+09:00", "pid": 23746, "process": {"exists": false, "pid": 23746}, "restart_count": 91, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.73, "sample_count": 1865, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 186225, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:23:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.045, "last_frame_pushed_at": "2026-07-06T19:23:06+09:00", "last_frame_read_age_sec": 0.038, "last_frame_read_at": "2026-07-06T19:23:06+09:00", "mjpeg": {"age_sec": 0.028, "exists": true, "mtime": "2026-07-06T19:23:06.762684+09:00", "mtime_epoch": 1783333386.762684, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156325}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:23:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.977, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.676, "pushed": 6.676, "read": 6.676}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.806, "count": 199, "max_ms": 10.077, "p95_ms": 8.056}, "gstreamer_push": {"avg_ms": 1.066, "count": 200, "max_ms": 1.631, "p95_ms": 1.23}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.033, "count": 199, "max_ms": 83.574, "p95_ms": 78.644}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.581, "count": 199, "max_ms": 117.499, "p95_ms": 103.438}, "yolo11_640_fire_11class": {"avg_ms": 60.116, "count": 199, "max_ms": 89.529, "p95_ms": 79.325}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.286, "count": 199, "max_ms": 108.244, "p95_ms": 96.382}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.447, "count": 200, "max_ms": 196.27, "p95_ms": 180.64}, "read_interval": {"avg_ms": 150.287, "count": 200, "max_ms": 204.381, "p95_ms": 188.618}, "triton_wait": {"avg_ms": 75.119, "count": 199, "max_ms": 117.511, "p95_ms": 103.675}, "visualize": {"avg_ms": 5.784, "count": 199, "max_ms": 9.657, "p95_ms": 7.023}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5946.418, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:23:06+09:00", "last_error_summary": "[h264 @ 0x55ef94605840] error while decoding MB 19 14, bytestream -18", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 303, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:23:06+09:00", "last_error_summary": "[h264 @ 0x55ef94605840] error while decoding MB 19 14, bytestream -18", "last_exit_at": "2026-07-06T19:23:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.872, "last_started_at": "2026-07-06T19:20:27+09:00", "pid": 23746, "process": {"exists": false, "pid": 23746}, "restart_count": 91, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.73, "sample_count": 1865, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 186226, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:23:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T19:23:06+09:00", "last_frame_read_age_sec": 0.179, "last_frame_read_at": "2026-07-06T19:23:06+09:00", "mjpeg": {"age_sec": 0.168, "exists": true, "mtime": "2026-07-06T19:23:06.762684+09:00", "mtime_epoch": 1783333386.762684, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156325}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:23:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.977, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.686, "pushed": 6.686, "read": 6.652}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.807, "count": 200, "max_ms": 10.077, "p95_ms": 7.969}, "gstreamer_push": {"avg_ms": 1.067, "count": 200, "max_ms": 1.631, "p95_ms": 1.23}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.065, "count": 200, "max_ms": 83.574, "p95_ms": 77.186}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.67, "count": 200, "max_ms": 117.499, "p95_ms": 103.063}, "yolo11_640_fire_11class": {"avg_ms": 60.122, "count": 200, "max_ms": 89.529, "p95_ms": 78.81}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.292, "count": 200, "max_ms": 108.244, "p95_ms": 96.112}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.62, "count": 200, "max_ms": 196.27, "p95_ms": 180.64}, "read_interval": {"avg_ms": 150.313, "count": 199, "max_ms": 204.381, "p95_ms": 189.975}, "triton_wait": {"avg_ms": 75.205, "count": 200, "max_ms": 117.511, "p95_ms": 103.451}, "visualize": {"avg_ms": 5.786, "count": 200, "max_ms": 9.657, "p95_ms": 6.962}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5946.41, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 303, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:23:06+09:00", "last_error_summary": "[h264 @ 0x55ef94605840] error while decoding MB 19 14, bytestream -18", "last_exit_at": "2026-07-06T19:23:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.877, "last_started_at": "2026-07-06T19:20:27+09:00", "pid": 23746, "process": {"exists": false, "pid": 23746}, "restart_count": 91, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.73, "sample_count": 1865, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 186226, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:23:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T19:23:06+09:00", "last_frame_read_age_sec": 0.183, "last_frame_read_at": "2026-07-06T19:23:06+09:00", "mjpeg": {"age_sec": 0.172, "exists": true, "mtime": "2026-07-06T19:23:06.762684+09:00", "mtime_epoch": 1783333386.762684, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156325}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:23:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.977, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.685, "pushed": 6.685, "read": 6.651}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.807, "count": 200, "max_ms": 10.077, "p95_ms": 7.969}, "gstreamer_push": {"avg_ms": 1.067, "count": 200, "max_ms": 1.631, "p95_ms": 1.23}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.065, "count": 200, "max_ms": 83.574, "p95_ms": 77.186}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.67, "count": 200, "max_ms": 117.499, "p95_ms": 103.063}, "yolo11_640_fire_11class": {"avg_ms": 60.122, "count": 200, "max_ms": 89.529, "p95_ms": 78.81}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.292, "count": 200, "max_ms": 108.244, "p95_ms": 96.112}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.62, "count": 200, "max_ms": 196.27, "p95_ms": 180.64}, "read_interval": {"avg_ms": 150.313, "count": 199, "max_ms": 204.381, "p95_ms": 189.975}, "triton_wait": {"avg_ms": 75.205, "count": 200, "max_ms": 117.511, "p95_ms": 103.451}, "visualize": {"avg_ms": 5.786, "count": 200, "max_ms": 9.657, "p95_ms": 6.962}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6049.215, "limit_mb": 8796093022207.996}, "detail": {"pid": 23853, "restart_count": 92, "start_count": 93}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 303, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:23:06+09:00", "last_error_summary": "[h264 @ 0x55ef94605840] error while decoding MB 19 14, bytestream -18", "last_exit_at": "2026-07-06T19:23:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:23:09+09:00", "pid": 23853, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.152, "name": "ffmpeg", "pid": 23853, "rss_mb": 139.152, "state": "S (sleeping)", "threads": 106}, "restart_count": 92, "restart_count_per_hour": 22, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.647, "sample_count": 1833, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 186226, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:23:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.853, "last_frame_pushed_at": "2026-07-06T19:23:06+09:00", "last_frame_read_age_sec": 5.028, "last_frame_read_at": "2026-07-06T19:23:06+09:00", "mjpeg": {"age_sec": 4.837, "exists": true, "mtime": "2026-07-06T19:23:06.942688+09:00", "mtime_epoch": 1783333386.9426885, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:23:11+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1626.977, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 169, "pushed_frames": 169, "read_frames": 168, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.668, "pushed": 5.668, "read": 5.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.78, "count": 169, "max_ms": 10.077, "p95_ms": 7.969}, "gstreamer_push": {"avg_ms": 1.068, "count": 169, "max_ms": 1.631, "p95_ms": 1.23}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.936, "count": 169, "max_ms": 83.574, "p95_ms": 78.664}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.167, "count": 169, "max_ms": 117.499, "p95_ms": 99.837}, "yolo11_640_fire_11class": {"avg_ms": 60.049, "count": 169, "max_ms": 89.529, "p95_ms": 79.723}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.49, "count": 169, "max_ms": 108.244, "p95_ms": 97.107}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.051, "count": 169, "max_ms": 196.27, "p95_ms": 175.662}, "read_interval": {"avg_ms": 148.724, "count": 168, "max_ms": 204.381, "p95_ms": 183.568}, "triton_wait": {"avg_ms": 74.708, "count": 169, "max_ms": 117.511, "p95_ms": 103.296}, "visualize": {"avg_ms": 5.732, "count": 169, "max_ms": 9.657, "p95_ms": 7.1}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.818, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1560}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.414, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55652f94b440] error while decoding MB 72 20, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 304, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:25:51+09:00", "last_error_summary": "[h264 @ 0x55652f94b440] error while decoding MB 72 20, bytestream -11", "last_exit_at": "2026-07-06T19:23:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.142, "last_started_at": "2026-07-06T19:23:09+09:00", "pid": 23853, "process": {"exists": false, "pid": 23853}, "restart_count": 92, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.665, "sample_count": 1966, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 187300, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:25:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T19:25:51+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T19:25:51+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T19:25:51.882482+09:00", "mtime_epoch": 1783333551.8824823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:25:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1627.492, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 209, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.935, "pushed": 6.935, "read": 6.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.701, "count": 208, "max_ms": 10.106, "p95_ms": 8.728}, "gstreamer_push": {"avg_ms": 1.077, "count": 208, "max_ms": 1.734, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.101, "count": 208, "max_ms": 79.834, "p95_ms": 73.628}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.639, "count": 208, "max_ms": 113.384, "p95_ms": 98.134}, "yolo11_640_fire_11class": {"avg_ms": 59.733, "count": 208, "max_ms": 87.144, "p95_ms": 78.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.228, "count": 208, "max_ms": 101.608, "p95_ms": 94.436}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.829, "count": 208, "max_ms": 198.767, "p95_ms": 172.325}, "read_interval": {"avg_ms": 143.877, "count": 209, "max_ms": 206.269, "p95_ms": 182.262}, "triton_wait": {"avg_ms": 73.795, "count": 208, "max_ms": 113.397, "p95_ms": 98.146}, "visualize": {"avg_ms": 5.585, "count": 208, "max_ms": 8.968, "p95_ms": 8.08}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1560}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5952.395, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:25:51+09:00", "last_error_summary": "[h264 @ 0x55652f94b440] error while decoding MB 72 20, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 304, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:25:51+09:00", "last_error_summary": "[h264 @ 0x55652f94b440] error while decoding MB 72 20, bytestream -11", "last_exit_at": "2026-07-06T19:25:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.272, "last_started_at": "2026-07-06T19:23:09+09:00", "pid": 23853, "process": {"exists": false, "pid": 23853}, "restart_count": 92, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.665, "sample_count": 1966, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 187301, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:25:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T19:25:52+09:00", "last_frame_read_age_sec": 0.18, "last_frame_read_at": "2026-07-06T19:25:51+09:00", "mjpeg": {"age_sec": 0.168, "exists": true, "mtime": "2026-07-06T19:25:51.882482+09:00", "mtime_epoch": 1783333551.8824823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:25:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1627.492, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 208, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.935, "pushed": 6.935, "read": 6.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.697, "count": 208, "max_ms": 10.106, "p95_ms": 8.728}, "gstreamer_push": {"avg_ms": 1.077, "count": 208, "max_ms": 1.734, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.214, "count": 208, "max_ms": 79.834, "p95_ms": 73.628}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.802, "count": 208, "max_ms": 113.384, "p95_ms": 98.134}, "yolo11_640_fire_11class": {"avg_ms": 59.881, "count": 208, "max_ms": 87.144, "p95_ms": 78.988}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.409, "count": 208, "max_ms": 101.608, "p95_ms": 94.436}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.094, "count": 208, "max_ms": 198.767, "p95_ms": 174.072}, "read_interval": {"avg_ms": 143.952, "count": 208, "max_ms": 206.269, "p95_ms": 182.262}, "triton_wait": {"avg_ms": 73.957, "count": 208, "max_ms": 113.397, "p95_ms": 98.146}, "visualize": {"avg_ms": 5.589, "count": 208, "max_ms": 8.968, "p95_ms": 8.08}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1560}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5952.387, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 304, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:25:51+09:00", "last_error_summary": "[h264 @ 0x55652f94b440] error while decoding MB 72 20, bytestream -11", "last_exit_at": "2026-07-06T19:25:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.276, "last_started_at": "2026-07-06T19:23:09+09:00", "pid": 23853, "process": {"exists": false, "pid": 23853}, "restart_count": 92, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.665, "sample_count": 1966, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 187301, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:25:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T19:25:52+09:00", "last_frame_read_age_sec": 0.184, "last_frame_read_at": "2026-07-06T19:25:51+09:00", "mjpeg": {"age_sec": 0.173, "exists": true, "mtime": "2026-07-06T19:25:51.882482+09:00", "mtime_epoch": 1783333551.8824823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:25:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1627.492, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 208, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.934, "pushed": 6.934, "read": 6.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.697, "count": 208, "max_ms": 10.106, "p95_ms": 8.728}, "gstreamer_push": {"avg_ms": 1.077, "count": 208, "max_ms": 1.734, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.214, "count": 208, "max_ms": 79.834, "p95_ms": 73.628}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.802, "count": 208, "max_ms": 113.384, "p95_ms": 98.134}, "yolo11_640_fire_11class": {"avg_ms": 59.881, "count": 208, "max_ms": 87.144, "p95_ms": 78.988}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.409, "count": 208, "max_ms": 101.608, "p95_ms": 94.436}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.094, "count": 208, "max_ms": 198.767, "p95_ms": 174.072}, "read_interval": {"avg_ms": 143.952, "count": 208, "max_ms": 206.269, "p95_ms": 182.262}, "triton_wait": {"avg_ms": 73.957, "count": 208, "max_ms": 113.397, "p95_ms": 98.146}, "visualize": {"avg_ms": 5.589, "count": 208, "max_ms": 8.968, "p95_ms": 8.08}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1560}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6059.074, "limit_mb": 8796093022207.996}, "detail": {"pid": 23960, "restart_count": 93, "start_count": 94}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 304, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:25:51+09:00", "last_error_summary": "[h264 @ 0x55652f94b440] error while decoding MB 72 20, bytestream -11", "last_exit_at": "2026-07-06T19:25:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:25:54+09:00", "pid": 23960, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.922, "name": "ffmpeg", "pid": 23960, "rss_mb": 138.922, "state": "S (sleeping)", "threads": 106}, "restart_count": 93, "restart_count_per_hour": 22, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.568, "sample_count": 1933, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 187301, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:25:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.954, "last_frame_pushed_at": "2026-07-06T19:25:52+09:00", "last_frame_read_age_sec": 5.13, "last_frame_read_at": "2026-07-06T19:25:51+09:00", "mjpeg": {"age_sec": 4.93, "exists": true, "mtime": "2026-07-06T19:25:52.070487+09:00", "mtime_epoch": 1783333552.0704865, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:25:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1627.492, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.816, "pushed": 5.816, "read": 5.783}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.685, "count": 174, "max_ms": 10.106, "p95_ms": 9.761}, "gstreamer_push": {"avg_ms": 1.08, "count": 174, "max_ms": 1.734, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.23, "count": 174, "max_ms": 77.855, "p95_ms": 73.752}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.916, "count": 174, "max_ms": 113.384, "p95_ms": 97.757}, "yolo11_640_fire_11class": {"avg_ms": 59.831, "count": 174, "max_ms": 86.712, "p95_ms": 78.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.513, "count": 174, "max_ms": 101.608, "p95_ms": 94.436}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.036, "count": 174, "max_ms": 198.767, "p95_ms": 174.072}, "read_interval": {"avg_ms": 143.945, "count": 173, "max_ms": 206.269, "p95_ms": 182.057}, "triton_wait": {"avg_ms": 74.065, "count": 174, "max_ms": 113.397, "p95_ms": 97.769}, "visualize": {"avg_ms": 5.615, "count": 174, "max_ms": 8.968, "p95_ms": 8.272}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1568}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5942.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556acc6c1940] error while decoding MB 73 31, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 305, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:28:26+09:00", "last_error_summary": "[h264 @ 0x556acc6c1940] error while decoding MB 73 31, bytestream -10", "last_exit_at": "2026-07-06T19:25:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 151.65, "last_started_at": "2026-07-06T19:25:54+09:00", "pid": 23960, "process": {"exists": false, "pid": 23960}, "restart_count": 93, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.021, "sample_count": 1921, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 188248, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:28:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-06T19:28:26+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T19:28:26+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T19:28:26.609915+09:00", "mtime_epoch": 1783333706.6099145, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163165}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:28:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1627.75, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.966, "pushed": 5.966, "read": 5.966}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.61, "count": 178, "max_ms": 11.183, "p95_ms": 9.68}, "gstreamer_push": {"avg_ms": 1.007, "count": 178, "max_ms": 1.421, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.058, "count": 178, "max_ms": 82.382, "p95_ms": 67.555}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.907, "count": 178, "max_ms": 112.469, "p95_ms": 99.055}, "yolo11_640_fire_11class": {"avg_ms": 57.454, "count": 178, "max_ms": 84.439, "p95_ms": 76.749}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.461, "count": 178, "max_ms": 102.713, "p95_ms": 94.473}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.188, "count": 178, "max_ms": 209.254, "p95_ms": 194.747}, "read_interval": {"avg_ms": 168.284, "count": 178, "max_ms": 216.46, "p95_ms": 201.998}, "triton_wait": {"avg_ms": 73.076, "count": 178, "max_ms": 112.482, "p95_ms": 99.068}, "visualize": {"avg_ms": 6.785, "count": 178, "max_ms": 11.889, "p95_ms": 8.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.835, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1568}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5947.469, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:28:26+09:00", "last_error_summary": "[h264 @ 0x556acc6c1940] error while decoding MB 73 31, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 305, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:28:26+09:00", "last_error_summary": "[h264 @ 0x556acc6c1940] error while decoding MB 73 31, bytestream -10", "last_exit_at": "2026-07-06T19:28:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 151.806, "last_started_at": "2026-07-06T19:25:54+09:00", "pid": 23960, "process": {"exists": false, "pid": 23960}, "restart_count": 93, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.094, "sample_count": 1921, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 188249, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:28:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T19:28:26+09:00", "last_frame_read_age_sec": 0.207, "last_frame_read_at": "2026-07-06T19:28:26+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-06T19:28:26.609915+09:00", "mtime_epoch": 1783333706.6099145, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163165}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:28:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1627.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.968, "pushed": 5.968, "read": 5.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.608, "count": 178, "max_ms": 11.183, "p95_ms": 9.68}, "gstreamer_push": {"avg_ms": 1.007, "count": 179, "max_ms": 1.421, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.208, "count": 178, "max_ms": 82.382, "p95_ms": 68.008}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.194, "count": 178, "max_ms": 118.531, "p95_ms": 100.548}, "yolo11_640_fire_11class": {"avg_ms": 57.559, "count": 178, "max_ms": 84.439, "p95_ms": 77.924}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.623, "count": 178, "max_ms": 102.713, "p95_ms": 94.828}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.416, "count": 179, "max_ms": 209.254, "p95_ms": 198.332}, "read_interval": {"avg_ms": 168.284, "count": 178, "max_ms": 216.46, "p95_ms": 201.998}, "triton_wait": {"avg_ms": 73.362, "count": 178, "max_ms": 118.544, "p95_ms": 100.559}, "visualize": {"avg_ms": 6.778, "count": 178, "max_ms": 11.889, "p95_ms": 8.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1568}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5950.086, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 305, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:28:26+09:00", "last_error_summary": "[h264 @ 0x556acc6c1940] error while decoding MB 73 31, bytestream -10", "last_exit_at": "2026-07-06T19:28:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 151.811, "last_started_at": "2026-07-06T19:25:54+09:00", "pid": 23960, "process": {"exists": false, "pid": 23960}, "restart_count": 93, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.094, "sample_count": 1921, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 188249, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:28:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T19:28:26+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-06T19:28:26+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-06T19:28:26.609915+09:00", "mtime_epoch": 1783333706.6099145, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163165}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:28:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1627.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.968, "pushed": 5.968, "read": 5.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.608, "count": 178, "max_ms": 11.183, "p95_ms": 9.68}, "gstreamer_push": {"avg_ms": 1.007, "count": 179, "max_ms": 1.421, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.208, "count": 178, "max_ms": 82.382, "p95_ms": 68.008}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.194, "count": 178, "max_ms": 118.531, "p95_ms": 100.548}, "yolo11_640_fire_11class": {"avg_ms": 57.559, "count": 178, "max_ms": 84.439, "p95_ms": 77.924}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.623, "count": 178, "max_ms": 102.713, "p95_ms": 94.828}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.416, "count": 179, "max_ms": 209.254, "p95_ms": 198.332}, "read_interval": {"avg_ms": 168.284, "count": 178, "max_ms": 216.46, "p95_ms": 201.998}, "triton_wait": {"avg_ms": 73.362, "count": 178, "max_ms": 118.544, "p95_ms": 100.559}, "visualize": {"avg_ms": 6.778, "count": 178, "max_ms": 11.889, "p95_ms": 8.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1568}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6060.57, "limit_mb": 8796093022207.996}, "detail": {"pid": 24067, "restart_count": 94, "start_count": 95}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 305, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:28:26+09:00", "last_error_summary": "[h264 @ 0x556acc6c1940] error while decoding MB 73 31, bytestream -10", "last_exit_at": "2026-07-06T19:28:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:28:28+09:00", "pid": 24067, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.473, "name": "ffmpeg", "pid": 24067, "rss_mb": 138.473, "state": "S (sleeping)", "threads": 106}, "restart_count": 94, "restart_count_per_hour": 22, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.021, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 188249, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:28:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.129, "last_frame_pushed_at": "2026-07-06T19:28:26+09:00", "last_frame_read_age_sec": 4.331, "last_frame_read_at": "2026-07-06T19:28:26+09:00", "mjpeg": {"age_sec": 4.109, "exists": true, "mtime": "2026-07-06T19:28:26.821919+09:00", "mtime_epoch": 1783333706.8219192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160011}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:28:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1627.75, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.21, "pushed": 5.21, "read": 5.177}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.581, "count": 155, "max_ms": 11.183, "p95_ms": 9.599}, "gstreamer_push": {"avg_ms": 1.011, "count": 156, "max_ms": 1.372, "p95_ms": 1.188}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.938, "count": 155, "max_ms": 82.382, "p95_ms": 68.008}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.052, "count": 155, "max_ms": 118.531, "p95_ms": 100.548}, "yolo11_640_fire_11class": {"avg_ms": 57.073, "count": 155, "max_ms": 84.439, "p95_ms": 77.924}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.064, "count": 155, "max_ms": 102.713, "p95_ms": 94.828}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.445, "count": 156, "max_ms": 209.254, "p95_ms": 192.251}, "read_interval": {"avg_ms": 166.304, "count": 155, "max_ms": 216.46, "p95_ms": 197.7}, "triton_wait": {"avg_ms": 73.183, "count": 155, "max_ms": 118.544, "p95_ms": 100.559}, "visualize": {"avg_ms": 6.674, "count": 155, "max_ms": 11.034, "p95_ms": 7.851}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1575}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5954.277, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557b2f8c2c40] error while decoding MB 57 31, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 306, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:30:41+09:00", "last_error_summary": "[h264 @ 0x557b2f8c2c40] error while decoding MB 57 31, bytestream -11", "last_exit_at": "2026-07-06T19:28:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.12, "last_started_at": "2026-07-06T19:28:28+09:00", "pid": 24067, "process": {"exists": false, "pid": 24067}, "restart_count": 94, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 190.704, "sample_count": 1822, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 189049, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:30:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.046, "last_frame_pushed_at": "2026-07-06T19:30:41+09:00", "last_frame_read_age_sec": 0.038, "last_frame_read_at": "2026-07-06T19:30:41+09:00", "mjpeg": {"age_sec": 0.021, "exists": true, "mtime": "2026-07-06T19:30:41.028822+09:00", "mtime_epoch": 1783333841.0288224, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160337}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:30:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1628.266, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 187, "pushed_frames": 187, "read_frames": 187, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.263, "pushed": 6.263, "read": 6.263}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.363, "count": 187, "max_ms": 10.367, "p95_ms": 8.933}, "gstreamer_push": {"avg_ms": 1.006, "count": 187, "max_ms": 1.433, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.592, "count": 187, "max_ms": 86.842, "p95_ms": 73.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.405, "count": 187, "max_ms": 115.078, "p95_ms": 98.674}, "yolo11_640_fire_11class": {"avg_ms": 59.715, "count": 187, "max_ms": 86.829, "p95_ms": 82.316}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.125, "count": 187, "max_ms": 103.99, "p95_ms": 92.668}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.957, "count": 187, "max_ms": 198.008, "p95_ms": 188.927}, "read_interval": {"avg_ms": 160.226, "count": 187, "max_ms": 207.718, "p95_ms": 196.769}, "triton_wait": {"avg_ms": 73.749, "count": 187, "max_ms": 115.09, "p95_ms": 99.905}, "visualize": {"avg_ms": 6.346, "count": 187, "max_ms": 9.637, "p95_ms": 7.601}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.859, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1575}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.695, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:30:41+09:00", "last_error_summary": "[h264 @ 0x557b2f8c2c40] error while decoding MB 57 31, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 306, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:30:41+09:00", "last_error_summary": "[h264 @ 0x557b2f8c2c40] error while decoding MB 57 31, bytestream -11", "last_exit_at": "2026-07-06T19:30:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.226, "last_started_at": "2026-07-06T19:28:28+09:00", "pid": 24067, "process": {"exists": false, "pid": 24067}, "restart_count": 94, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 190.704, "sample_count": 1822, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 189050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:30:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T19:30:41+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-06T19:30:41+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-06T19:30:41.028822+09:00", "mtime_epoch": 1783333841.0288224, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160337}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:30:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1628.266, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 187, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.274, "pushed": 6.274, "read": 6.241}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.37, "count": 187, "max_ms": 10.367, "p95_ms": 8.933}, "gstreamer_push": {"avg_ms": 1.005, "count": 188, "max_ms": 1.433, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.521, "count": 187, "max_ms": 86.842, "p95_ms": 73.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.379, "count": 187, "max_ms": 115.078, "p95_ms": 98.674}, "yolo11_640_fire_11class": {"avg_ms": 59.678, "count": 187, "max_ms": 86.829, "p95_ms": 82.316}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.089, "count": 187, "max_ms": 103.99, "p95_ms": 92.668}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.886, "count": 188, "max_ms": 198.008, "p95_ms": 188.927}, "read_interval": {"avg_ms": 160.226, "count": 187, "max_ms": 207.718, "p95_ms": 196.769}, "triton_wait": {"avg_ms": 73.723, "count": 187, "max_ms": 115.09, "p95_ms": 99.905}, "visualize": {"avg_ms": 6.349, "count": 187, "max_ms": 9.637, "p95_ms": 7.601}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1575}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.812, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 306, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:30:41+09:00", "last_error_summary": "[h264 @ 0x557b2f8c2c40] error while decoding MB 57 31, bytestream -11", "last_exit_at": "2026-07-06T19:30:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.23, "last_started_at": "2026-07-06T19:28:28+09:00", "pid": 24067, "process": {"exists": false, "pid": 24067}, "restart_count": 94, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 190.704, "sample_count": 1822, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 189050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:30:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T19:30:41+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-06T19:30:41+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-06T19:30:41.028822+09:00", "mtime_epoch": 1783333841.0288224, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160337}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:30:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1628.266, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 187, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.273, "pushed": 6.273, "read": 6.24}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.37, "count": 187, "max_ms": 10.367, "p95_ms": 8.933}, "gstreamer_push": {"avg_ms": 1.005, "count": 188, "max_ms": 1.433, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.521, "count": 187, "max_ms": 86.842, "p95_ms": 73.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.379, "count": 187, "max_ms": 115.078, "p95_ms": 98.674}, "yolo11_640_fire_11class": {"avg_ms": 59.678, "count": 187, "max_ms": 86.829, "p95_ms": 82.316}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.089, "count": 187, "max_ms": 103.99, "p95_ms": 92.668}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.886, "count": 188, "max_ms": 198.008, "p95_ms": 188.927}, "read_interval": {"avg_ms": 160.226, "count": 187, "max_ms": 207.718, "p95_ms": 196.769}, "triton_wait": {"avg_ms": 73.723, "count": 187, "max_ms": 115.09, "p95_ms": 99.905}, "visualize": {"avg_ms": 6.349, "count": 187, "max_ms": 9.637, "p95_ms": 7.601}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1575}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6062.023, "limit_mb": 8796093022207.996}, "detail": {"pid": 24174, "restart_count": 95, "start_count": 96}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 306, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:30:41+09:00", "last_error_summary": "[h264 @ 0x557b2f8c2c40] error while decoding MB 57 31, bytestream -11", "last_exit_at": "2026-07-06T19:30:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:30:43+09:00", "pid": 24174, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.285, "name": "ffmpeg", "pid": 24174, "rss_mb": 139.285, "state": "S (sleeping)", "threads": 106}, "restart_count": 95, "restart_count_per_hour": 22, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 96}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 191.039, "sample_count": 1793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 189050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:30:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.399, "last_frame_pushed_at": "2026-07-06T19:30:41+09:00", "last_frame_read_age_sec": 4.539, "last_frame_read_at": "2026-07-06T19:30:41+09:00", "mjpeg": {"age_sec": 4.39, "exists": true, "mtime": "2026-07-06T19:30:41.160825+09:00", "mtime_epoch": 1783333841.1608253, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160658}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:30:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1628.266, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.354, "pushed": 5.354, "read": 5.321}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.366, "count": 160, "max_ms": 9.716, "p95_ms": 8.89}, "gstreamer_push": {"avg_ms": 0.999, "count": 160, "max_ms": 1.433, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.041, "count": 160, "max_ms": 84.555, "p95_ms": 72.044}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.955, "count": 160, "max_ms": 112.377, "p95_ms": 94.822}, "yolo11_640_fire_11class": {"avg_ms": 59.306, "count": 160, "max_ms": 86.829, "p95_ms": 80.937}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.664, "count": 160, "max_ms": 102.718, "p95_ms": 92.587}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.041, "count": 160, "max_ms": 198.008, "p95_ms": 188.927}, "read_interval": {"avg_ms": 160.388, "count": 159, "max_ms": 207.718, "p95_ms": 198.452}, "triton_wait": {"avg_ms": 73.329, "count": 160, "max_ms": 112.39, "p95_ms": 94.836}, "visualize": {"avg_ms": 6.366, "count": 160, "max_ms": 9.637, "p95_ms": 7.619}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5959.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 307, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:33:15+09:00", "last_error_summary": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "last_exit_at": "2026-07-06T19:30:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.245, "last_started_at": "2026-07-06T19:30:43+09:00", "pid": 24174, "process": {"exists": false, "pid": 24174}, "restart_count": 95, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 96}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.018, "sample_count": 1832, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 190016, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:33:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T19:33:15+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T19:33:15+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T19:33:15.752106+09:00", "mtime_epoch": 1783333995.752106, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:33:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1628.781, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.334, "pushed": 6.334, "read": 6.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.994, "count": 190, "max_ms": 10.306, "p95_ms": 9.092}, "gstreamer_push": {"avg_ms": 1.029, "count": 190, "max_ms": 1.275, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.476, "count": 190, "max_ms": 91.411, "p95_ms": 75.614}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.019, "count": 190, "max_ms": 124.031, "p95_ms": 101.503}, "yolo11_640_fire_11class": {"avg_ms": 59.975, "count": 190, "max_ms": 92.19, "p95_ms": 79.29}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.255, "count": 190, "max_ms": 103.244, "p95_ms": 93.884}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.255, "count": 190, "max_ms": 207.554, "p95_ms": 183.182}, "read_interval": {"avg_ms": 157.631, "count": 191, "max_ms": 215.045, "p95_ms": 190.446}, "triton_wait": {"avg_ms": 75.481, "count": 190, "max_ms": 124.043, "p95_ms": 101.516}, "visualize": {"avg_ms": 6.074, "count": 190, "max_ms": 8.465, "p95_ms": 7.318}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5958.273, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:33:15+09:00", "last_error_summary": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 307, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:33:15+09:00", "last_error_summary": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "last_exit_at": "2026-07-06T19:33:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.341, "last_started_at": "2026-07-06T19:30:43+09:00", "pid": 24174, "process": {"exists": false, "pid": 24174}, "restart_count": 95, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 96}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.018, "sample_count": 1832, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 190017, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:33:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T19:33:15+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-06T19:33:15+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-06T19:33:15.752106+09:00", "mtime_epoch": 1783333995.752106, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:33:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1628.781, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.385, "pushed": 6.385, "read": 6.352}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.992, "count": 191, "max_ms": 10.306, "p95_ms": 9.092}, "gstreamer_push": {"avg_ms": 1.03, "count": 191, "max_ms": 1.275, "p95_ms": 1.173}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.532, "count": 191, "max_ms": 91.411, "p95_ms": 75.614}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.063, "count": 191, "max_ms": 124.031, "p95_ms": 101.503}, "yolo11_640_fire_11class": {"avg_ms": 60.003, "count": 191, "max_ms": 92.19, "p95_ms": 79.29}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.316, "count": 191, "max_ms": 103.244, "p95_ms": 93.884}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.199, "count": 191, "max_ms": 207.554, "p95_ms": 183.182}, "read_interval": {"avg_ms": 157.61, "count": 190, "max_ms": 215.045, "p95_ms": 190.446}, "triton_wait": {"avg_ms": 75.523, "count": 191, "max_ms": 124.043, "p95_ms": 101.516}, "visualize": {"avg_ms": 6.069, "count": 191, "max_ms": 8.465, "p95_ms": 7.318}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5958.266, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 307, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:33:15+09:00", "last_error_summary": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "last_exit_at": "2026-07-06T19:33:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.346, "last_started_at": "2026-07-06T19:30:43+09:00", "pid": 24174, "process": {"exists": false, "pid": 24174}, "restart_count": 95, "restart_count_per_hour": 21, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 96}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.018, "sample_count": 1832, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 190017, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:33:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T19:33:15+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T19:33:15+09:00", "mjpeg": {"age_sec": 0.144, "exists": true, "mtime": "2026-07-06T19:33:15.752106+09:00", "mtime_epoch": 1783333995.752106, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:33:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1628.781, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.384, "pushed": 6.384, "read": 6.35}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.992, "count": 191, "max_ms": 10.306, "p95_ms": 9.092}, "gstreamer_push": {"avg_ms": 1.03, "count": 191, "max_ms": 1.275, "p95_ms": 1.173}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.532, "count": 191, "max_ms": 91.411, "p95_ms": 75.614}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.063, "count": 191, "max_ms": 124.031, "p95_ms": 101.503}, "yolo11_640_fire_11class": {"avg_ms": 60.003, "count": 191, "max_ms": 92.19, "p95_ms": 79.29}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.316, "count": 191, "max_ms": 103.244, "p95_ms": 93.884}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.199, "count": 191, "max_ms": 207.554, "p95_ms": 183.182}, "read_interval": {"avg_ms": 157.61, "count": 190, "max_ms": 215.045, "p95_ms": 190.446}, "triton_wait": {"avg_ms": 75.523, "count": 191, "max_ms": 124.043, "p95_ms": 101.516}, "visualize": {"avg_ms": 6.069, "count": 191, "max_ms": 8.465, "p95_ms": 7.318}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6068.797, "limit_mb": 8796093022207.996}, "detail": {"pid": 24281, "restart_count": 96, "start_count": 97}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 307, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:33:15+09:00", "last_error_summary": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "last_exit_at": "2026-07-06T19:33:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:33:18+09:00", "pid": 24281, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.789, "name": "ffmpeg", "pid": 24281, "rss_mb": 149.789, "state": "S (sleeping)", "threads": 106}, "restart_count": 96, "restart_count_per_hour": 22, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 97}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.927, "sample_count": 1808, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 190017, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:33:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.212, "last_frame_pushed_at": "2026-07-06T19:33:15+09:00", "last_frame_read_age_sec": 4.352, "last_frame_read_at": "2026-07-06T19:33:15+09:00", "mjpeg": {"age_sec": 4.197, "exists": true, "mtime": "2026-07-06T19:33:15.900109+09:00", "mtime_epoch": 1783333995.900109, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169499}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:33:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1628.781, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 165, "pushed_frames": 165, "read_frames": 164, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.521, "pushed": 5.521, "read": 5.487}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.954, "count": 165, "max_ms": 10.306, "p95_ms": 9.068}, "gstreamer_push": {"avg_ms": 1.031, "count": 165, "max_ms": 1.275, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.588, "count": 165, "max_ms": 91.411, "p95_ms": 77.281}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.778, "count": 165, "max_ms": 124.031, "p95_ms": 100.54}, "yolo11_640_fire_11class": {"avg_ms": 59.972, "count": 165, "max_ms": 92.19, "p95_ms": 80.256}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.4, "count": 165, "max_ms": 103.244, "p95_ms": 94.12}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.29, "count": 165, "max_ms": 207.554, "p95_ms": 182.114}, "read_interval": {"avg_ms": 156.755, "count": 164, "max_ms": 215.045, "p95_ms": 190.388}, "triton_wait": {"avg_ms": 75.304, "count": 165, "max_ms": 124.043, "p95_ms": 100.554}, "visualize": {"avg_ms": 6.024, "count": 165, "max_ms": 8.465, "p95_ms": 7.318}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1590}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5959.707, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:33:15+09:00", "last_error_summary": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 307, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:33:15+09:00", "last_error_summary": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "last_exit_at": "2026-07-06T19:35:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.475, "last_started_at": "2026-07-06T19:33:18+09:00", "pid": 24281, "process": {"exists": false, "pid": 24281}, "restart_count": 96, "restart_count_per_hour": 22, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 97}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.251, "sample_count": 1862, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 190912, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:35:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T19:35:44+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-06T19:35:44+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-06T19:35:44.435213+09:00", "mtime_epoch": 1783334144.4352129, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174996}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:35:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1629.039, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 187, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.294, "pushed": 6.294, "read": 6.26}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.165, "count": 188, "max_ms": 10.246, "p95_ms": 9.01}, "gstreamer_push": {"avg_ms": 1.017, "count": 188, "max_ms": 1.412, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.035, "count": 188, "max_ms": 80.603, "p95_ms": 71.95}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.048, "count": 188, "max_ms": 109.883, "p95_ms": 98.702}, "yolo11_640_fire_11class": {"avg_ms": 58.479, "count": 188, "max_ms": 84.732, "p95_ms": 76.475}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.584, "count": 188, "max_ms": 98.659, "p95_ms": 91.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.246, "count": 188, "max_ms": 193.676, "p95_ms": 177.119}, "read_interval": {"avg_ms": 159.954, "count": 187, "max_ms": 202.448, "p95_ms": 186.761}, "triton_wait": {"avg_ms": 72.29, "count": 188, "max_ms": 109.895, "p95_ms": 98.713}, "visualize": {"avg_ms": 6.353, "count": 188, "max_ms": 8.178, "p95_ms": 7.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1590}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5959.699, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 307, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:33:15+09:00", "last_error_summary": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "last_exit_at": "2026-07-06T19:35:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.48, "last_started_at": "2026-07-06T19:33:18+09:00", "pid": 24281, "process": {"exists": false, "pid": 24281}, "restart_count": 96, "restart_count_per_hour": 22, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 97}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.251, "sample_count": 1862, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 190912, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:35:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T19:35:44+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T19:35:44+09:00", "mjpeg": {"age_sec": 0.141, "exists": true, "mtime": "2026-07-06T19:35:44.435213+09:00", "mtime_epoch": 1783334144.4352129, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174996}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:35:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1629.039, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 187, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.293, "pushed": 6.293, "read": 6.259}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.165, "count": 188, "max_ms": 10.246, "p95_ms": 9.01}, "gstreamer_push": {"avg_ms": 1.017, "count": 188, "max_ms": 1.412, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.035, "count": 188, "max_ms": 80.603, "p95_ms": 71.95}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.048, "count": 188, "max_ms": 109.883, "p95_ms": 98.702}, "yolo11_640_fire_11class": {"avg_ms": 58.479, "count": 188, "max_ms": 84.732, "p95_ms": 76.475}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.584, "count": 188, "max_ms": 98.659, "p95_ms": 91.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.246, "count": 188, "max_ms": 193.676, "p95_ms": 177.119}, "read_interval": {"avg_ms": 159.954, "count": 187, "max_ms": 202.448, "p95_ms": 186.761}, "triton_wait": {"avg_ms": 72.29, "count": 188, "max_ms": 109.895, "p95_ms": 98.713}, "visualize": {"avg_ms": 6.353, "count": 188, "max_ms": 8.178, "p95_ms": 7.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1590}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6066.953, "limit_mb": 8796093022207.996}, "detail": {"pid": 24388, "restart_count": 97, "start_count": 98}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 307, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:33:15+09:00", "last_error_summary": "[h264 @ 0x55e569681d40] error while decoding MB 74 26, bytestream -5", "last_exit_at": "2026-07-06T19:35:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:35:46+09:00", "pid": 24388, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.055, "name": "ffmpeg", "pid": 24388, "rss_mb": 139.055, "state": "S (sleeping)", "threads": 106}, "restart_count": 97, "restart_count_per_hour": 23, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 98}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.427, "sample_count": 1843, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 190912, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:35:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.07, "last_frame_pushed_at": "2026-07-06T19:35:44+09:00", "last_frame_read_age_sec": 4.212, "last_frame_read_at": "2026-07-06T19:35:44+09:00", "mjpeg": {"age_sec": 4.049, "exists": true, "mtime": "2026-07-06T19:35:44.587216+09:00", "mtime_epoch": 1783334144.587216, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175006}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:35:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1629.039, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 163, "pushed_frames": 163, "read_frames": 162, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.446, "pushed": 5.446, "read": 5.412}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.163, "count": 163, "max_ms": 10.246, "p95_ms": 9.01}, "gstreamer_push": {"avg_ms": 1.02, "count": 163, "max_ms": 1.412, "p95_ms": 1.172}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.144, "count": 163, "max_ms": 80.603, "p95_ms": 72.37}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.072, "count": 163, "max_ms": 109.883, "p95_ms": 98.702}, "yolo11_640_fire_11class": {"avg_ms": 58.784, "count": 163, "max_ms": 84.732, "p95_ms": 76.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.836, "count": 163, "max_ms": 98.659, "p95_ms": 91.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.098, "count": 163, "max_ms": 193.676, "p95_ms": 176.048}, "read_interval": {"avg_ms": 159.643, "count": 162, "max_ms": 202.448, "p95_ms": 186.655}, "triton_wait": {"avg_ms": 72.349, "count": 163, "max_ms": 109.895, "p95_ms": 98.713}, "visualize": {"avg_ms": 6.352, "count": 163, "max_ms": 8.178, "p95_ms": 7.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1597}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5963.23, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560f509d1240] cabac decode of qscale diff failed at 64 41 [h264 @ 0x560f509d1240] error while decoding MB 64 41, bytestream 1540 [h264 @ 0x560f50b6ee80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f50b6ee80] error while decoding MB 0 42, bytestream 44174 [h264 @ 0x560f50d0cb80] cabac decode of qscale diff failed at 25 44 [h264 @ 0x560f50d0cb80] error while decoding MB 25 44, bytestream 4717 [h264 @ 0x560f50c3dd00] left block unavailable for requested intra4x4 mod", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 313, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:37:59+09:00", "last_error_summary": "[h264 @ 0x560f509d1240] cabac decode of qscale diff failed at 64 41 [h264 @ 0x560f509d1240] error while decoding MB 64 41, bytestream 1540 [h264 @ 0x560f50b6ee80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f50b6ee80] error while decoding MB 0 42, bytestream 44174 [h264 @ 0x560f50d0cb80] cabac decode of qscale diff failed at 25 44 [h264 @ 0x560f50d0cb80] error while decoding MB 25 44, bytestream 4717 [h264 @ 0x560f50c3dd00] left block unavailable for requested intra4x4 mod", "last_exit_at": "2026-07-06T19:35:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.793, "last_started_at": "2026-07-06T19:35:46+09:00", "pid": 24388, "process": {"exists": false, "pid": 24388}, "restart_count": 97, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 98}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.991, "sample_count": 1786, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 191696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:37:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.042, "last_frame_pushed_at": "2026-07-06T19:37:59+09:00", "last_frame_read_age_sec": 0.036, "last_frame_read_at": "2026-07-06T19:37:59+09:00", "mjpeg": {"age_sec": 0.023, "exists": true, "mtime": "2026-07-06T19:37:59.406002+09:00", "mtime_epoch": 1783334279.4060023, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:37:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1629.555, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.07, "pushed": 4.07, "read": 4.07}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.451, "count": 121, "max_ms": 9.183, "p95_ms": 7.488}, "gstreamer_push": {"avg_ms": 1.076, "count": 122, "max_ms": 1.304, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.502, "count": 121, "max_ms": 79.059, "p95_ms": 74.339}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.836, "count": 121, "max_ms": 105.798, "p95_ms": 99.139}, "yolo11_640_fire_11class": {"avg_ms": 59.809, "count": 121, "max_ms": 83.327, "p95_ms": 76.662}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.46, "count": 121, "max_ms": 100.993, "p95_ms": 91.324}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.133, "count": 122, "max_ms": 187.089, "p95_ms": 165.915}, "read_interval": {"avg_ms": 246.388, "count": 122, "max_ms": 12631.635, "p95_ms": 175.475}, "triton_wait": {"avg_ms": 73.444, "count": 121, "max_ms": 105.811, "p95_ms": 99.149}, "visualize": {"avg_ms": 5.575, "count": 121, "max_ms": 9.509, "p95_ms": 6.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1597}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5961.73, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:37:59+09:00", "last_error_summary": "[h264 @ 0x560f509d1240] cabac decode of qscale diff failed at 64 41 [h264 @ 0x560f509d1240] error while decoding MB 64 41, bytestream 1540 [h264 @ 0x560f50b6ee80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f50b6ee80] error while decoding MB 0 42, bytestream 44174 [h264 @ 0x560f50d0cb80] cabac decode of qscale diff failed at 25 44 [h264 @ 0x560f50d0cb80] error while decoding MB 25 44, bytestream 4717 [h264 @ 0x560f50c3dd00] left block unavailable for requested intra4x4 mod", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 313, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:37:59+09:00", "last_error_summary": "[h264 @ 0x560f509d1240] cabac decode of qscale diff failed at 64 41 [h264 @ 0x560f509d1240] error while decoding MB 64 41, bytestream 1540 [h264 @ 0x560f50b6ee80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f50b6ee80] error while decoding MB 0 42, bytestream 44174 [h264 @ 0x560f50d0cb80] cabac decode of qscale diff failed at 25 44 [h264 @ 0x560f50d0cb80] error while decoding MB 25 44, bytestream 4717 [h264 @ 0x560f50c3dd00] left block unavailable for requested intra4x4 mod", "last_exit_at": "2026-07-06T19:37:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.916, "last_started_at": "2026-07-06T19:35:46+09:00", "pid": 24388, "process": {"exists": false, "pid": 24388}, "restart_count": 97, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 98}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.991, "sample_count": 1787, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 191697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:37:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T19:37:59+09:00", "last_frame_read_age_sec": 0.158, "last_frame_read_at": "2026-07-06T19:37:59+09:00", "mjpeg": {"age_sec": 0.145, "exists": true, "mtime": "2026-07-06T19:37:59.406002+09:00", "mtime_epoch": 1783334279.4060023, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:37:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1629.555, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.069, "pushed": 4.069, "read": 4.036}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.454, "count": 121, "max_ms": 9.183, "p95_ms": 7.488}, "gstreamer_push": {"avg_ms": 1.077, "count": 122, "max_ms": 1.304, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.561, "count": 121, "max_ms": 79.059, "p95_ms": 74.339}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.116, "count": 121, "max_ms": 105.798, "p95_ms": 99.139}, "yolo11_640_fire_11class": {"avg_ms": 59.972, "count": 121, "max_ms": 83.327, "p95_ms": 76.662}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.791, "count": 121, "max_ms": 100.993, "p95_ms": 91.691}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.424, "count": 122, "max_ms": 187.089, "p95_ms": 165.915}, "read_interval": {"avg_ms": 247.389, "count": 121, "max_ms": 12631.635, "p95_ms": 175.475}, "triton_wait": {"avg_ms": 73.761, "count": 121, "max_ms": 105.811, "p95_ms": 99.229}, "visualize": {"avg_ms": 5.576, "count": 121, "max_ms": 9.509, "p95_ms": 6.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1597}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5961.723, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 313, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:37:59+09:00", "last_error_summary": "[h264 @ 0x560f509d1240] cabac decode of qscale diff failed at 64 41 [h264 @ 0x560f509d1240] error while decoding MB 64 41, bytestream 1540 [h264 @ 0x560f50b6ee80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f50b6ee80] error while decoding MB 0 42, bytestream 44174 [h264 @ 0x560f50d0cb80] cabac decode of qscale diff failed at 25 44 [h264 @ 0x560f50d0cb80] error while decoding MB 25 44, bytestream 4717 [h264 @ 0x560f50c3dd00] left block unavailable for requested intra4x4 mod", "last_exit_at": "2026-07-06T19:37:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.921, "last_started_at": "2026-07-06T19:35:46+09:00", "pid": 24388, "process": {"exists": false, "pid": 24388}, "restart_count": 97, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 98}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.991, "sample_count": 1787, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 191697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:37:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T19:37:59+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-06T19:37:59+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-06T19:37:59.406002+09:00", "mtime_epoch": 1783334279.4060023, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:37:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1629.555, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.069, "pushed": 4.069, "read": 4.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.454, "count": 121, "max_ms": 9.183, "p95_ms": 7.488}, "gstreamer_push": {"avg_ms": 1.077, "count": 122, "max_ms": 1.304, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.561, "count": 121, "max_ms": 79.059, "p95_ms": 74.339}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.116, "count": 121, "max_ms": 105.798, "p95_ms": 99.139}, "yolo11_640_fire_11class": {"avg_ms": 59.972, "count": 121, "max_ms": 83.327, "p95_ms": 76.662}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.791, "count": 121, "max_ms": 100.993, "p95_ms": 91.691}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.424, "count": 122, "max_ms": 187.089, "p95_ms": 165.915}, "read_interval": {"avg_ms": 247.389, "count": 121, "max_ms": 12631.635, "p95_ms": 175.475}, "triton_wait": {"avg_ms": 73.761, "count": 121, "max_ms": 105.811, "p95_ms": 99.229}, "visualize": {"avg_ms": 5.576, "count": 121, "max_ms": 9.509, "p95_ms": 6.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1597}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6069.855, "limit_mb": 8796093022207.996}, "detail": {"pid": 24495, "restart_count": 98, "start_count": 99}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 313, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:37:59+09:00", "last_error_summary": "[h264 @ 0x560f509d1240] cabac decode of qscale diff failed at 64 41 [h264 @ 0x560f509d1240] error while decoding MB 64 41, bytestream 1540 [h264 @ 0x560f50b6ee80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560f50b6ee80] error while decoding MB 0 42, bytestream 44174 [h264 @ 0x560f50d0cb80] cabac decode of qscale diff failed at 25 44 [h264 @ 0x560f50d0cb80] error while decoding MB 25 44, bytestream 4717 [h264 @ 0x560f50c3dd00] left block unavailable for requested intra4x4 mod", "last_exit_at": "2026-07-06T19:38:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:38:01+09:00", "pid": 24495, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.395, "name": "ffmpeg", "pid": 24495, "rss_mb": 139.395, "state": "S (sleeping)", "threads": 106}, "restart_count": 98, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 99}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.991, "sample_count": 1758, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 191697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:37:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.242, "last_frame_pushed_at": "2026-07-06T19:37:59+09:00", "last_frame_read_age_sec": 4.396, "last_frame_read_at": "2026-07-06T19:37:59+09:00", "mjpeg": {"age_sec": 4.227, "exists": true, "mtime": "2026-07-06T19:37:59.562006+09:00", "mtime_epoch": 1783334279.5620055, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180028}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:38:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1629.555, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 90, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.045, "pushed": 3.045, "read": 3.012}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.564, "count": 91, "max_ms": 9.183, "p95_ms": 7.692}, "gstreamer_push": {"avg_ms": 1.066, "count": 91, "max_ms": 1.304, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.657, "count": 91, "max_ms": 79.059, "p95_ms": 74.721}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.867, "count": 91, "max_ms": 105.798, "p95_ms": 99.139}, "yolo11_640_fire_11class": {"avg_ms": 60.177, "count": 91, "max_ms": 83.327, "p95_ms": 76.724}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.5, "count": 91, "max_ms": 100.993, "p95_ms": 92.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.567, "count": 91, "max_ms": 187.089, "p95_ms": 168.248}, "read_interval": {"avg_ms": 284.802, "count": 90, "max_ms": 12631.635, "p95_ms": 177.448}, "triton_wait": {"avg_ms": 73.41, "count": 91, "max_ms": 105.811, "p95_ms": 99.229}, "visualize": {"avg_ms": 5.639, "count": 91, "max_ms": 9.509, "p95_ms": 6.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.484, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5588139ee740] error while decoding MB 66 9, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 314, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:40:45+09:00", "last_error_summary": "[h264 @ 0x5588139ee740] error while decoding MB 66 9, bytestream -5", "last_exit_at": "2026-07-06T19:38:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.54, "last_started_at": "2026-07-06T19:38:01+09:00", "pid": 24495, "process": {"exists": false, "pid": 24495}, "restart_count": 98, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 99}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.728, "sample_count": 1876, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 192788, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:40:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-06T19:40:45+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T19:40:45+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-06T19:40:45.269400+09:00", "mtime_epoch": 1783334445.2694004, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180754}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:40:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1630.07, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 209, "pushed_frames": 209, "read_frames": 209, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.982, "pushed": 6.982, "read": 6.982}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.453, "count": 208, "max_ms": 10.498, "p95_ms": 7.766}, "gstreamer_push": {"avg_ms": 1.07, "count": 209, "max_ms": 1.289, "p95_ms": 1.2}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.672, "count": 208, "max_ms": 82.622, "p95_ms": 73.732}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.885, "count": 208, "max_ms": 131.582, "p95_ms": 102.512}, "yolo11_640_fire_11class": {"avg_ms": 59.321, "count": 208, "max_ms": 89.202, "p95_ms": 78.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.61, "count": 208, "max_ms": 100.273, "p95_ms": 93.378}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.533, "count": 209, "max_ms": 215.063, "p95_ms": 175.079}, "read_interval": {"avg_ms": 143.662, "count": 209, "max_ms": 223.926, "p95_ms": 184.46}, "triton_wait": {"avg_ms": 74.203, "count": 208, "max_ms": 131.598, "p95_ms": 102.525}, "visualize": {"avg_ms": 5.471, "count": 208, "max_ms": 8.643, "p95_ms": 6.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5956.324, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:40:45+09:00", "last_error_summary": "[h264 @ 0x5588139ee740] error while decoding MB 66 9, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 314, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:40:45+09:00", "last_error_summary": "[h264 @ 0x5588139ee740] error while decoding MB 66 9, bytestream -5", "last_exit_at": "2026-07-06T19:40:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.642, "last_started_at": "2026-07-06T19:38:01+09:00", "pid": 24495, "process": {"exists": false, "pid": 24495}, "restart_count": 98, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 99}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.728, "sample_count": 1877, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 192789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:40:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T19:40:45+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-06T19:40:45+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-06T19:40:45.269400+09:00", "mtime_epoch": 1783334445.2694004, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180754}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:40:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1630.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 209, "pushed_frames": 209, "read_frames": 208, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.993, "pushed": 6.993, "read": 6.959}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.46, "count": 209, "max_ms": 10.498, "p95_ms": 7.827}, "gstreamer_push": {"avg_ms": 1.069, "count": 209, "max_ms": 1.289, "p95_ms": 1.2}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.645, "count": 209, "max_ms": 82.622, "p95_ms": 73.732}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.847, "count": 209, "max_ms": 131.582, "p95_ms": 102.512}, "yolo11_640_fire_11class": {"avg_ms": 59.281, "count": 209, "max_ms": 89.202, "p95_ms": 78.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.576, "count": 209, "max_ms": 100.273, "p95_ms": 93.378}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.556, "count": 209, "max_ms": 215.063, "p95_ms": 175.079}, "read_interval": {"avg_ms": 143.621, "count": 208, "max_ms": 223.926, "p95_ms": 184.46}, "triton_wait": {"avg_ms": 74.163, "count": 209, "max_ms": 131.598, "p95_ms": 102.525}, "visualize": {"avg_ms": 5.475, "count": 209, "max_ms": 8.643, "p95_ms": 6.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5958.812, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 314, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:40:45+09:00", "last_error_summary": "[h264 @ 0x5588139ee740] error while decoding MB 66 9, bytestream -5", "last_exit_at": "2026-07-06T19:40:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.649, "last_started_at": "2026-07-06T19:38:01+09:00", "pid": 24495, "process": {"exists": false, "pid": 24495}, "restart_count": 98, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 99}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.728, "sample_count": 1877, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 192789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:40:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T19:40:45+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-06T19:40:45+09:00", "mjpeg": {"age_sec": 0.167, "exists": true, "mtime": "2026-07-06T19:40:45.269400+09:00", "mtime_epoch": 1783334445.2694004, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180754}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:40:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1630.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 209, "pushed_frames": 209, "read_frames": 208, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.991, "pushed": 6.991, "read": 6.958}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.46, "count": 209, "max_ms": 10.498, "p95_ms": 7.827}, "gstreamer_push": {"avg_ms": 1.069, "count": 209, "max_ms": 1.289, "p95_ms": 1.2}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.645, "count": 209, "max_ms": 82.622, "p95_ms": 73.732}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.847, "count": 209, "max_ms": 131.582, "p95_ms": 102.512}, "yolo11_640_fire_11class": {"avg_ms": 59.281, "count": 209, "max_ms": 89.202, "p95_ms": 78.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.576, "count": 209, "max_ms": 100.273, "p95_ms": 93.378}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.556, "count": 209, "max_ms": 215.063, "p95_ms": 175.079}, "read_interval": {"avg_ms": 143.621, "count": 208, "max_ms": 223.926, "p95_ms": 184.46}, "triton_wait": {"avg_ms": 74.163, "count": 209, "max_ms": 131.598, "p95_ms": 102.525}, "visualize": {"avg_ms": 5.475, "count": 209, "max_ms": 8.643, "p95_ms": 6.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6066.934, "limit_mb": 8796093022207.996}, "detail": {"pid": 24602, "restart_count": 99, "start_count": 100}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 314, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:40:45+09:00", "last_error_summary": "[h264 @ 0x5588139ee740] error while decoding MB 66 9, bytestream -5", "last_exit_at": "2026-07-06T19:40:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:40:47+09:00", "pid": 24602, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.473, "name": "ffmpeg", "pid": 24602, "rss_mb": 139.473, "state": "S (sleeping)", "threads": 106}, "restart_count": 99, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 100}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.728, "sample_count": 1872, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 192789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:40:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.153, "last_frame_pushed_at": "2026-07-06T19:40:45+09:00", "last_frame_read_age_sec": 4.303, "last_frame_read_at": "2026-07-06T19:40:45+09:00", "mjpeg": {"age_sec": 4.145, "exists": true, "mtime": "2026-07-06T19:40:45.433404+09:00", "mtime_epoch": 1783334445.4334037, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172661}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:40:49+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1630.07, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 182, "pushed_frames": 182, "read_frames": 181, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.07, "pushed": 6.07, "read": 6.037}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.468, "count": 181, "max_ms": 10.498, "p95_ms": 7.766}, "gstreamer_push": {"avg_ms": 1.08, "count": 182, "max_ms": 1.289, "p95_ms": 1.201}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.149, "count": 181, "max_ms": 82.622, "p95_ms": 72.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.615, "count": 181, "max_ms": 131.582, "p95_ms": 101.373}, "yolo11_640_fire_11class": {"avg_ms": 58.982, "count": 181, "max_ms": 89.202, "p95_ms": 77.557}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.21, "count": 181, "max_ms": 100.273, "p95_ms": 92.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.446, "count": 182, "max_ms": 196.151, "p95_ms": 168.537}, "read_interval": {"avg_ms": 142.57, "count": 181, "max_ms": 204.516, "p95_ms": 176.44}, "triton_wait": {"avg_ms": 73.939, "count": 181, "max_ms": 131.598, "p95_ms": 101.387}, "visualize": {"avg_ms": 5.482, "count": 181, "max_ms": 8.643, "p95_ms": 6.741}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5968.586, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563a790d6c00] error while decoding MB 13 18, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 315, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:43:43+09:00", "last_error_summary": "[h264 @ 0x563a790d6c00] error while decoding MB 13 18, bytestream -5", "last_exit_at": "2026-07-06T19:40:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 175.751, "last_started_at": "2026-07-06T19:40:47+09:00", "pid": 24602, "process": {"exists": false, "pid": 24602}, "restart_count": 99, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 100}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.281, "sample_count": 2034, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 193993, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:43:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-06T19:43:43+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T19:43:43+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T19:43:43.281020+09:00", "mtime_epoch": 1783334623.2810204, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178107}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:43:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1630.844, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 199, "pushed_frames": 199, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.662, "pushed": 6.662, "read": 6.662}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.904, "count": 199, "max_ms": 10.814, "p95_ms": 8.934}, "gstreamer_push": {"avg_ms": 1.072, "count": 199, "max_ms": 1.996, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.592, "count": 199, "max_ms": 79.399, "p95_ms": 74.185}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.529, "count": 199, "max_ms": 115.477, "p95_ms": 100.568}, "yolo11_640_fire_11class": {"avg_ms": 60.355, "count": 199, "max_ms": 87.585, "p95_ms": 82.102}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.51, "count": 199, "max_ms": 102.466, "p95_ms": 92.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.566, "count": 199, "max_ms": 194.713, "p95_ms": 177.905}, "read_interval": {"avg_ms": 150.501, "count": 199, "max_ms": 204.955, "p95_ms": 186.536}, "triton_wait": {"avg_ms": 74.81, "count": 199, "max_ms": 115.49, "p95_ms": 102.055}, "visualize": {"avg_ms": 5.823, "count": 199, "max_ms": 8.502, "p95_ms": 7.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5967.801, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:43:43+09:00", "last_error_summary": "[h264 @ 0x563a790d6c00] error while decoding MB 13 18, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 315, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:43:43+09:00", "last_error_summary": "[h264 @ 0x563a790d6c00] error while decoding MB 13 18, bytestream -5", "last_exit_at": "2026-07-06T19:43:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 175.825, "last_started_at": "2026-07-06T19:40:47+09:00", "pid": 24602, "process": {"exists": false, "pid": 24602}, "restart_count": 99, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 100}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.281, "sample_count": 2034, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 193994, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:43:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T19:43:43+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-06T19:43:43+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T19:43:43.281020+09:00", "mtime_epoch": 1783334623.2810204, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178107}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:43:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1630.844, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.679, "pushed": 6.679, "read": 6.646}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.912, "count": 200, "max_ms": 10.814, "p95_ms": 8.916}, "gstreamer_push": {"avg_ms": 1.072, "count": 200, "max_ms": 1.996, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.55, "count": 200, "max_ms": 79.399, "p95_ms": 74.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.466, "count": 200, "max_ms": 115.477, "p95_ms": 100.386}, "yolo11_640_fire_11class": {"avg_ms": 60.302, "count": 200, "max_ms": 87.585, "p95_ms": 80.181}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.44, "count": 200, "max_ms": 102.466, "p95_ms": 92.527}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.458, "count": 200, "max_ms": 194.713, "p95_ms": 177.248}, "read_interval": {"avg_ms": 150.501, "count": 199, "max_ms": 204.955, "p95_ms": 186.536}, "triton_wait": {"avg_ms": 74.746, "count": 200, "max_ms": 115.49, "p95_ms": 100.577}, "visualize": {"avg_ms": 5.827, "count": 200, "max_ms": 8.502, "p95_ms": 7.051}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5967.793, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 315, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:43:43+09:00", "last_error_summary": "[h264 @ 0x563a790d6c00] error while decoding MB 13 18, bytestream -5", "last_exit_at": "2026-07-06T19:43:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 175.831, "last_started_at": "2026-07-06T19:40:47+09:00", "pid": 24602, "process": {"exists": false, "pid": 24602}, "restart_count": 99, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 100}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.281, "sample_count": 2034, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 193994, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:43:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T19:43:43+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T19:43:43+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-06T19:43:43.281020+09:00", "mtime_epoch": 1783334623.2810204, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178107}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:43:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1630.844, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.678, "pushed": 6.678, "read": 6.645}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.912, "count": 200, "max_ms": 10.814, "p95_ms": 8.916}, "gstreamer_push": {"avg_ms": 1.072, "count": 200, "max_ms": 1.996, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.55, "count": 200, "max_ms": 79.399, "p95_ms": 74.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.466, "count": 200, "max_ms": 115.477, "p95_ms": 100.386}, "yolo11_640_fire_11class": {"avg_ms": 60.302, "count": 200, "max_ms": 87.585, "p95_ms": 80.181}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.44, "count": 200, "max_ms": 102.466, "p95_ms": 92.527}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.458, "count": 200, "max_ms": 194.713, "p95_ms": 177.248}, "read_interval": {"avg_ms": 150.501, "count": 199, "max_ms": 204.955, "p95_ms": 186.536}, "triton_wait": {"avg_ms": 74.746, "count": 200, "max_ms": 115.49, "p95_ms": 100.577}, "visualize": {"avg_ms": 5.827, "count": 200, "max_ms": 8.502, "p95_ms": 7.051}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6076.344, "limit_mb": 8796093022207.996}, "detail": {"pid": 24709, "restart_count": 100, "start_count": 101}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 315, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:43:43+09:00", "last_error_summary": "[h264 @ 0x563a790d6c00] error while decoding MB 13 18, bytestream -5", "last_exit_at": "2026-07-06T19:43:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:43:45+09:00", "pid": 24709, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.207, "name": "ffmpeg", "pid": 24709, "rss_mb": 139.207, "state": "S (sleeping)", "threads": 106}, "restart_count": 100, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 101}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.281, "sample_count": 2005, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 193994, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:43:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.447, "last_frame_pushed_at": "2026-07-06T19:43:43+09:00", "last_frame_read_age_sec": 4.569, "last_frame_read_at": "2026-07-06T19:43:43+09:00", "mjpeg": {"age_sec": 4.432, "exists": true, "mtime": "2026-07-06T19:43:43.409023+09:00", "mtime_epoch": 1783334623.409023, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176695}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:43:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1630.844, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 169, "pushed_frames": 169, "read_frames": 168, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.649, "pushed": 5.649, "read": 5.616}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.983, "count": 169, "max_ms": 10.814, "p95_ms": 8.934}, "gstreamer_push": {"avg_ms": 1.073, "count": 169, "max_ms": 1.996, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.58, "count": 169, "max_ms": 79.399, "p95_ms": 74.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.604, "count": 169, "max_ms": 115.477, "p95_ms": 100.568}, "yolo11_640_fire_11class": {"avg_ms": 60.516, "count": 169, "max_ms": 87.585, "p95_ms": 82.193}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.643, "count": 169, "max_ms": 102.466, "p95_ms": 93.286}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.625, "count": 169, "max_ms": 194.713, "p95_ms": 177.905}, "read_interval": {"avg_ms": 151.672, "count": 168, "max_ms": 204.955, "p95_ms": 186.536}, "triton_wait": {"avg_ms": 74.909, "count": 169, "max_ms": 115.49, "p95_ms": 102.055}, "visualize": {"avg_ms": 5.876, "count": 169, "max_ms": 8.502, "p95_ms": 7.051}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1625}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5961.379, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5576c8f3efc0] error while decoding MB 14 41, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 316, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:46:24+09:00", "last_error_summary": "[h264 @ 0x5576c8f3efc0] error while decoding MB 14 41, bytestream -9", "last_exit_at": "2026-07-06T19:43:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.284, "last_started_at": "2026-07-06T19:43:45+09:00", "pid": 24709, "process": {"exists": false, "pid": 24709}, "restart_count": 100, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 101}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.447, "sample_count": 1994, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 195026, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:46:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.052, "last_frame_pushed_at": "2026-07-06T19:46:24+09:00", "last_frame_read_age_sec": 0.045, "last_frame_read_at": "2026-07-06T19:46:24+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-06T19:46:24.088273+09:00", "mtime_epoch": 1783334784.0882733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:46:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1631.102, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.871, "pushed": 6.871, "read": 6.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.599, "count": 205, "max_ms": 10.682, "p95_ms": 8.283}, "gstreamer_push": {"avg_ms": 1.061, "count": 206, "max_ms": 1.521, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.605, "count": 205, "max_ms": 86.878, "p95_ms": 72.441}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.403, "count": 205, "max_ms": 116.532, "p95_ms": 99.584}, "yolo11_640_fire_11class": {"avg_ms": 60.092, "count": 205, "max_ms": 87.518, "p95_ms": 82.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.639, "count": 205, "max_ms": 101.523, "p95_ms": 93.67}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.063, "count": 206, "max_ms": 191.452, "p95_ms": 171.496}, "read_interval": {"avg_ms": 145.942, "count": 206, "max_ms": 198.48, "p95_ms": 179.317}, "triton_wait": {"avg_ms": 73.738, "count": 205, "max_ms": 116.549, "p95_ms": 99.597}, "visualize": {"avg_ms": 5.659, "count": 205, "max_ms": 8.495, "p95_ms": 7.238}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1625}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5962.977, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:46:24+09:00", "last_error_summary": "[h264 @ 0x5576c8f3efc0] error while decoding MB 14 41, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 316, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:46:24+09:00", "last_error_summary": "[h264 @ 0x5576c8f3efc0] error while decoding MB 14 41, bytestream -9", "last_exit_at": "2026-07-06T19:46:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.368, "last_started_at": "2026-07-06T19:43:45+09:00", "pid": 24709, "process": {"exists": false, "pid": 24709}, "restart_count": 100, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 101}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.447, "sample_count": 1995, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 195027, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:46:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T19:46:24+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T19:46:24+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T19:46:24.088273+09:00", "mtime_epoch": 1783334784.0882733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:46:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1631.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.887, "pushed": 6.887, "read": 6.853}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.596, "count": 206, "max_ms": 10.682, "p95_ms": 8.283}, "gstreamer_push": {"avg_ms": 1.06, "count": 206, "max_ms": 1.521, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.618, "count": 206, "max_ms": 86.878, "p95_ms": 72.441}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.371, "count": 206, "max_ms": 116.532, "p95_ms": 99.584}, "yolo11_640_fire_11class": {"avg_ms": 60.078, "count": 206, "max_ms": 87.518, "p95_ms": 82.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.618, "count": 206, "max_ms": 101.523, "p95_ms": 93.67}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.037, "count": 206, "max_ms": 191.452, "p95_ms": 171.496}, "read_interval": {"avg_ms": 145.983, "count": 205, "max_ms": 198.48, "p95_ms": 179.317}, "triton_wait": {"avg_ms": 73.705, "count": 206, "max_ms": 116.549, "p95_ms": 99.597}, "visualize": {"avg_ms": 5.656, "count": 206, "max_ms": 8.495, "p95_ms": 7.238}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1625}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5963.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 316, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:46:24+09:00", "last_error_summary": "[h264 @ 0x5576c8f3efc0] error while decoding MB 14 41, bytestream -9", "last_exit_at": "2026-07-06T19:46:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.374, "last_started_at": "2026-07-06T19:43:45+09:00", "pid": 24709, "process": {"exists": false, "pid": 24709}, "restart_count": 100, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 101}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.447, "sample_count": 1995, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 195027, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:46:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T19:46:24+09:00", "last_frame_read_age_sec": 0.135, "last_frame_read_at": "2026-07-06T19:46:24+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-06T19:46:24.088273+09:00", "mtime_epoch": 1783334784.0882733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:46:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1631.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.885, "pushed": 6.885, "read": 6.852}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.596, "count": 206, "max_ms": 10.682, "p95_ms": 8.283}, "gstreamer_push": {"avg_ms": 1.06, "count": 206, "max_ms": 1.521, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.618, "count": 206, "max_ms": 86.878, "p95_ms": 72.441}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.371, "count": 206, "max_ms": 116.532, "p95_ms": 99.584}, "yolo11_640_fire_11class": {"avg_ms": 60.078, "count": 206, "max_ms": 87.518, "p95_ms": 82.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.618, "count": 206, "max_ms": 101.523, "p95_ms": 93.67}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.037, "count": 206, "max_ms": 191.452, "p95_ms": 171.496}, "read_interval": {"avg_ms": 145.983, "count": 205, "max_ms": 198.48, "p95_ms": 179.317}, "triton_wait": {"avg_ms": 73.705, "count": 206, "max_ms": 116.549, "p95_ms": 99.597}, "visualize": {"avg_ms": 5.656, "count": 206, "max_ms": 8.495, "p95_ms": 7.238}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1625}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6074.117, "limit_mb": 8796093022207.996}, "detail": {"pid": 24816, "restart_count": 101, "start_count": 102}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 316, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:46:24+09:00", "last_error_summary": "[h264 @ 0x5576c8f3efc0] error while decoding MB 14 41, bytestream -9", "last_exit_at": "2026-07-06T19:46:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:46:26+09:00", "pid": 24816, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.008, "name": "ffmpeg", "pid": 24816, "rss_mb": 142.008, "state": "S (sleeping)", "threads": 106}, "restart_count": 101, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 102}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.592, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 195027, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:46:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.48, "last_frame_pushed_at": "2026-07-06T19:46:24+09:00", "last_frame_read_age_sec": 4.605, "last_frame_read_at": "2026-07-06T19:46:24+09:00", "mjpeg": {"age_sec": 4.464, "exists": true, "mtime": "2026-07-06T19:46:24.220276+09:00", "mtime_epoch": 1783334784.220276, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179472}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:46:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1631.102, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.907, "pushed": 5.907, "read": 5.873}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.516, "count": 176, "max_ms": 10.682, "p95_ms": 8.119}, "gstreamer_push": {"avg_ms": 1.061, "count": 177, "max_ms": 1.521, "p95_ms": 1.217}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.724, "count": 176, "max_ms": 86.878, "p95_ms": 72.957}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.37, "count": 176, "max_ms": 116.532, "p95_ms": 100.006}, "yolo11_640_fire_11class": {"avg_ms": 60.34, "count": 176, "max_ms": 87.518, "p95_ms": 84.148}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.814, "count": 176, "max_ms": 101.523, "p95_ms": 94.009}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.039, "count": 177, "max_ms": 191.452, "p95_ms": 170.216}, "read_interval": {"avg_ms": 145.036, "count": 176, "max_ms": 198.48, "p95_ms": 178.614}, "triton_wait": {"avg_ms": 73.736, "count": 176, "max_ms": 116.549, "p95_ms": 100.03}, "visualize": {"avg_ms": 5.602, "count": 176, "max_ms": 8.495, "p95_ms": 7.34}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5966.023, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ae13443480] error while decoding MB 49 30, bytestream -22", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 317, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:49:03+09:00", "last_error_summary": "[h264 @ 0x55ae13443480] error while decoding MB 49 30, bytestream -22", "last_exit_at": "2026-07-06T19:46:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 157.314, "last_started_at": "2026-07-06T19:46:26+09:00", "pid": 24816, "process": {"exists": false, "pid": 24816}, "restart_count": 101, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 102}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.603, "sample_count": 1939, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 196037, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:49:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T19:49:03+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T19:49:03+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-06T19:49:03.959496+09:00", "mtime_epoch": 1783334943.9594955, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175625}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:49:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1631.617, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.147, "pushed": 6.147, "read": 6.147}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.18, "count": 184, "max_ms": 10.922, "p95_ms": 8.968}, "gstreamer_push": {"avg_ms": 1.023, "count": 184, "max_ms": 1.818, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.209, "count": 184, "max_ms": 89.399, "p95_ms": 73.644}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.944, "count": 184, "max_ms": 108.542, "p95_ms": 100.464}, "yolo11_640_fire_11class": {"avg_ms": 59.91, "count": 184, "max_ms": 95.213, "p95_ms": 81.26}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.583, "count": 184, "max_ms": 106.062, "p95_ms": 91.792}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.639, "count": 184, "max_ms": 193.322, "p95_ms": 185.625}, "read_interval": {"avg_ms": 163.099, "count": 184, "max_ms": 201.63, "p95_ms": 194.15}, "triton_wait": {"avg_ms": 75.177, "count": 184, "max_ms": 108.556, "p95_ms": 101.583}, "visualize": {"avg_ms": 6.482, "count": 184, "max_ms": 8.678, "p95_ms": 7.914}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5964.988, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:49:03+09:00", "last_error_summary": "[h264 @ 0x55ae13443480] error while decoding MB 49 30, bytestream -22", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 317, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:49:03+09:00", "last_error_summary": "[h264 @ 0x55ae13443480] error while decoding MB 49 30, bytestream -22", "last_exit_at": "2026-07-06T19:49:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 157.418, "last_started_at": "2026-07-06T19:46:26+09:00", "pid": 24816, "process": {"exists": false, "pid": 24816}, "restart_count": 101, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 102}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.603, "sample_count": 1939, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 196038, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:49:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T19:49:04+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-06T19:49:03+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T19:49:03.959496+09:00", "mtime_epoch": 1783334943.9594955, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175625}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:49:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1631.617, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.157, "pushed": 6.157, "read": 6.123}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.184, "count": 184, "max_ms": 10.922, "p95_ms": 8.968}, "gstreamer_push": {"avg_ms": 1.023, "count": 184, "max_ms": 1.818, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.266, "count": 184, "max_ms": 89.399, "p95_ms": 73.644}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.028, "count": 184, "max_ms": 108.542, "p95_ms": 100.464}, "yolo11_640_fire_11class": {"avg_ms": 60.001, "count": 184, "max_ms": 95.213, "p95_ms": 81.26}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.663, "count": 184, "max_ms": 106.062, "p95_ms": 91.792}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.752, "count": 184, "max_ms": 193.322, "p95_ms": 185.625}, "read_interval": {"avg_ms": 163.211, "count": 183, "max_ms": 201.63, "p95_ms": 194.15}, "triton_wait": {"avg_ms": 75.261, "count": 184, "max_ms": 108.556, "p95_ms": 101.583}, "visualize": {"avg_ms": 6.489, "count": 184, "max_ms": 8.678, "p95_ms": 7.914}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5964.98, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 317, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:49:03+09:00", "last_error_summary": "[h264 @ 0x55ae13443480] error while decoding MB 49 30, bytestream -22", "last_exit_at": "2026-07-06T19:49:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 157.424, "last_started_at": "2026-07-06T19:46:26+09:00", "pid": 24816, "process": {"exists": false, "pid": 24816}, "restart_count": 101, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 102}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.603, "sample_count": 1939, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 196038, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:49:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T19:49:04+09:00", "last_frame_read_age_sec": 0.164, "last_frame_read_at": "2026-07-06T19:49:03+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-06T19:49:03.959496+09:00", "mtime_epoch": 1783334943.9594955, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175625}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:49:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1631.617, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.155, "pushed": 6.155, "read": 6.122}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.184, "count": 184, "max_ms": 10.922, "p95_ms": 8.968}, "gstreamer_push": {"avg_ms": 1.023, "count": 184, "max_ms": 1.818, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.266, "count": 184, "max_ms": 89.399, "p95_ms": 73.644}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.028, "count": 184, "max_ms": 108.542, "p95_ms": 100.464}, "yolo11_640_fire_11class": {"avg_ms": 60.001, "count": 184, "max_ms": 95.213, "p95_ms": 81.26}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.663, "count": 184, "max_ms": 106.062, "p95_ms": 91.792}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.752, "count": 184, "max_ms": 193.322, "p95_ms": 185.625}, "read_interval": {"avg_ms": 163.211, "count": 183, "max_ms": 201.63, "p95_ms": 194.15}, "triton_wait": {"avg_ms": 75.261, "count": 184, "max_ms": 108.556, "p95_ms": 101.583}, "visualize": {"avg_ms": 6.489, "count": 184, "max_ms": 8.678, "p95_ms": 7.914}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6075.594, "limit_mb": 8796093022207.996}, "detail": {"pid": 24923, "restart_count": 102, "start_count": 103}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 317, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:49:03+09:00", "last_error_summary": "[h264 @ 0x55ae13443480] error while decoding MB 49 30, bytestream -22", "last_exit_at": "2026-07-06T19:49:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:49:06+09:00", "pid": 24923, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.109, "name": "ffmpeg", "pid": 24923, "rss_mb": 141.109, "state": "S (sleeping)", "threads": 106}, "restart_count": 102, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 103}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.825, "sample_count": 1906, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 196038, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:49:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.542, "last_frame_pushed_at": "2026-07-06T19:49:04+09:00", "last_frame_read_age_sec": 4.696, "last_frame_read_at": "2026-07-06T19:49:03+09:00", "mjpeg": {"age_sec": 4.524, "exists": true, "mtime": "2026-07-06T19:49:04.115499+09:00", "mtime_epoch": 1783334944.1154988, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173185}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:49:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1631.617, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.215, "pushed": 5.215, "read": 5.182}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.175, "count": 156, "max_ms": 10.922, "p95_ms": 8.968}, "gstreamer_push": {"avg_ms": 1.023, "count": 156, "max_ms": 1.818, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.494, "count": 156, "max_ms": 89.399, "p95_ms": 74.713}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.138, "count": 156, "max_ms": 108.542, "p95_ms": 100.464}, "yolo11_640_fire_11class": {"avg_ms": 59.92, "count": 156, "max_ms": 95.213, "p95_ms": 81.26}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.508, "count": 156, "max_ms": 104.494, "p95_ms": 91.792}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.134, "count": 156, "max_ms": 193.322, "p95_ms": 186.305}, "read_interval": {"avg_ms": 163.631, "count": 155, "max_ms": 201.63, "p95_ms": 195.366}, "triton_wait": {"avg_ms": 75.392, "count": 156, "max_ms": 108.556, "p95_ms": 101.866}, "visualize": {"avg_ms": 6.492, "count": 156, "max_ms": 8.678, "p95_ms": 7.914}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5968.562, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c30b23b540] error while decoding MB 21 16, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 318, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:51:46+09:00", "last_error_summary": "[h264 @ 0x55c30b23b540] error while decoding MB 21 16, bytestream -10", "last_exit_at": "2026-07-06T19:49:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.173, "last_started_at": "2026-07-06T19:49:06+09:00", "pid": 24923, "process": {"exists": false, "pid": 24923}, "restart_count": 102, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 103}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.943, "sample_count": 1942, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 197088, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:51:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.048, "last_frame_pushed_at": "2026-07-06T19:51:46+09:00", "last_frame_read_age_sec": 0.041, "last_frame_read_at": "2026-07-06T19:51:46+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-06T19:51:46.779988+09:00", "mtime_epoch": 1783335106.7799876, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:51:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1632.133, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.803, "pushed": 6.803, "read": 6.803}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.754, "count": 203, "max_ms": 10.778, "p95_ms": 8.988}, "gstreamer_push": {"avg_ms": 1.09, "count": 204, "max_ms": 1.432, "p95_ms": 1.279}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.083, "count": 203, "max_ms": 87.237, "p95_ms": 72.659}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.926, "count": 203, "max_ms": 117.24, "p95_ms": 97.84}, "yolo11_640_fire_11class": {"avg_ms": 60.752, "count": 203, "max_ms": 90.78, "p95_ms": 79.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.287, "count": 203, "max_ms": 98.086, "p95_ms": 91.821}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.215, "count": 204, "max_ms": 197.008, "p95_ms": 173.137}, "read_interval": {"avg_ms": 147.371, "count": 204, "max_ms": 205.001, "p95_ms": 182.63}, "triton_wait": {"avg_ms": 74.287, "count": 203, "max_ms": 117.253, "p95_ms": 97.852}, "visualize": {"avg_ms": 5.768, "count": 203, "max_ms": 9.317, "p95_ms": 7.111}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5966.988, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:51:46+09:00", "last_error_summary": "[h264 @ 0x55c30b23b540] error while decoding MB 21 16, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 318, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:51:46+09:00", "last_error_summary": "[h264 @ 0x55c30b23b540] error while decoding MB 21 16, bytestream -10", "last_exit_at": "2026-07-06T19:51:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.295, "last_started_at": "2026-07-06T19:49:06+09:00", "pid": 24923, "process": {"exists": false, "pid": 24923}, "restart_count": 102, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 103}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.943, "sample_count": 1942, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 197089, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:51:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T19:51:46+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-06T19:51:46+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-06T19:51:46.779988+09:00", "mtime_epoch": 1783335106.7799876, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:51:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1632.133, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.805, "pushed": 6.805, "read": 6.772}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.752, "count": 203, "max_ms": 10.778, "p95_ms": 8.988}, "gstreamer_push": {"avg_ms": 1.091, "count": 204, "max_ms": 1.432, "p95_ms": 1.279}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.2, "count": 203, "max_ms": 87.237, "p95_ms": 73.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.065, "count": 203, "max_ms": 117.24, "p95_ms": 97.84}, "yolo11_640_fire_11class": {"avg_ms": 60.868, "count": 203, "max_ms": 90.78, "p95_ms": 79.346}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.461, "count": 203, "max_ms": 98.086, "p95_ms": 92.12}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.404, "count": 204, "max_ms": 197.008, "p95_ms": 173.137}, "read_interval": {"avg_ms": 147.478, "count": 203, "max_ms": 205.001, "p95_ms": 182.63}, "triton_wait": {"avg_ms": 74.43, "count": 203, "max_ms": 117.253, "p95_ms": 97.852}, "visualize": {"avg_ms": 5.769, "count": 203, "max_ms": 9.317, "p95_ms": 7.111}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5958.363, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 318, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:51:46+09:00", "last_error_summary": "[h264 @ 0x55c30b23b540] error while decoding MB 21 16, bytestream -10", "last_exit_at": "2026-07-06T19:51:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.3, "last_started_at": "2026-07-06T19:49:06+09:00", "pid": 24923, "process": {"exists": false, "pid": 24923}, "restart_count": 102, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 103}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.943, "sample_count": 1942, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 197089, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:51:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T19:51:46+09:00", "last_frame_read_age_sec": 0.168, "last_frame_read_at": "2026-07-06T19:51:46+09:00", "mjpeg": {"age_sec": 0.159, "exists": true, "mtime": "2026-07-06T19:51:46.779988+09:00", "mtime_epoch": 1783335106.7799876, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:51:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1632.133, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.804, "pushed": 6.804, "read": 6.77}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.752, "count": 203, "max_ms": 10.778, "p95_ms": 8.988}, "gstreamer_push": {"avg_ms": 1.091, "count": 204, "max_ms": 1.432, "p95_ms": 1.279}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.2, "count": 203, "max_ms": 87.237, "p95_ms": 73.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.065, "count": 203, "max_ms": 117.24, "p95_ms": 97.84}, "yolo11_640_fire_11class": {"avg_ms": 60.868, "count": 203, "max_ms": 90.78, "p95_ms": 79.346}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.461, "count": 203, "max_ms": 98.086, "p95_ms": 92.12}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.404, "count": 204, "max_ms": 197.008, "p95_ms": 173.137}, "read_interval": {"avg_ms": 147.478, "count": 203, "max_ms": 205.001, "p95_ms": 182.63}, "triton_wait": {"avg_ms": 74.43, "count": 203, "max_ms": 117.253, "p95_ms": 97.852}, "visualize": {"avg_ms": 5.769, "count": 203, "max_ms": 9.317, "p95_ms": 7.111}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6075.16, "limit_mb": 8796093022207.996}, "detail": {"pid": 25030, "restart_count": 103, "start_count": 104}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 318, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:51:46+09:00", "last_error_summary": "[h264 @ 0x55c30b23b540] error while decoding MB 21 16, bytestream -10", "last_exit_at": "2026-07-06T19:51:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:51:49+09:00", "pid": 25030, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.77, "name": "ffmpeg", "pid": 25030, "rss_mb": 138.77, "state": "S (sleeping)", "threads": 106}, "restart_count": 103, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 104}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.266, "sample_count": 1907, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 197089, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:51:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.729, "last_frame_pushed_at": "2026-07-06T19:51:46+09:00", "last_frame_read_age_sec": 4.886, "last_frame_read_at": "2026-07-06T19:51:46+09:00", "mjpeg": {"age_sec": 4.713, "exists": true, "mtime": "2026-07-06T19:51:46.943988+09:00", "mtime_epoch": 1783335106.9439883, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181148}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:51:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1632.133, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 170, "pushed_frames": 170, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.685, "pushed": 5.685, "read": 5.651}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.711, "count": 170, "max_ms": 10.778, "p95_ms": 8.688}, "gstreamer_push": {"avg_ms": 1.088, "count": 170, "max_ms": 1.429, "p95_ms": 1.279}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.078, "count": 170, "max_ms": 87.237, "p95_ms": 73.667}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.136, "count": 170, "max_ms": 117.24, "p95_ms": 97.84}, "yolo11_640_fire_11class": {"avg_ms": 60.861, "count": 170, "max_ms": 90.78, "p95_ms": 80.523}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.61, "count": 170, "max_ms": 98.086, "p95_ms": 92.253}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.783, "count": 170, "max_ms": 197.008, "p95_ms": 174.465}, "read_interval": {"avg_ms": 148.776, "count": 169, "max_ms": 205.001, "p95_ms": 183.369}, "triton_wait": {"avg_ms": 74.539, "count": 170, "max_ms": 117.253, "p95_ms": 97.852}, "visualize": {"avg_ms": 5.809, "count": 170, "max_ms": 9.317, "p95_ms": 7.165}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5969.609, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c918f007c0] error while decoding MB 75 22, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 319, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:54:33+09:00", "last_error_summary": "[h264 @ 0x55c918f007c0] error while decoding MB 75 22, bytestream -5", "last_exit_at": "2026-07-06T19:51:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.714, "last_started_at": "2026-07-06T19:51:49+09:00", "pid": 25030, "process": {"exists": false, "pid": 25030}, "restart_count": 103, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 104}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.545, "sample_count": 1968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 198168, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:54:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T19:54:33+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T19:54:33+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T19:54:33.329107+09:00", "mtime_epoch": 1783335273.3291066, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178877}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:54:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1632.648, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 199, "pushed_frames": 199, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.646, "pushed": 6.646, "read": 6.646}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.877, "count": 198, "max_ms": 9.949, "p95_ms": 8.574}, "gstreamer_push": {"avg_ms": 1.051, "count": 199, "max_ms": 1.701, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.731, "count": 198, "max_ms": 86.205, "p95_ms": 77.283}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.764, "count": 198, "max_ms": 115.774, "p95_ms": 103.493}, "yolo11_640_fire_11class": {"avg_ms": 60.606, "count": 198, "max_ms": 89.032, "p95_ms": 82.538}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.191, "count": 198, "max_ms": 105.253, "p95_ms": 99.081}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.13, "count": 199, "max_ms": 199.27, "p95_ms": 181.083}, "read_interval": {"avg_ms": 150.91, "count": 199, "max_ms": 207.897, "p95_ms": 188.888}, "triton_wait": {"avg_ms": 74.23, "count": 198, "max_ms": 115.785, "p95_ms": 103.504}, "visualize": {"avg_ms": 5.87, "count": 198, "max_ms": 8.217, "p95_ms": 7.048}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5968.801, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:54:33+09:00", "last_error_summary": "[h264 @ 0x55c918f007c0] error while decoding MB 75 22, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 319, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:54:33+09:00", "last_error_summary": "[h264 @ 0x55c918f007c0] error while decoding MB 75 22, bytestream -5", "last_exit_at": "2026-07-06T19:54:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.849, "last_started_at": "2026-07-06T19:51:49+09:00", "pid": 25030, "process": {"exists": false, "pid": 25030}, "restart_count": 103, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 104}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.545, "sample_count": 1968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 198169, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:54:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T19:54:33+09:00", "last_frame_read_age_sec": 0.186, "last_frame_read_at": "2026-07-06T19:54:33+09:00", "mjpeg": {"age_sec": 0.176, "exists": true, "mtime": "2026-07-06T19:54:33.329107+09:00", "mtime_epoch": 1783335273.3291066, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178877}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:54:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1632.648, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 199, "pushed_frames": 199, "read_frames": 198, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.65, "pushed": 6.65, "read": 6.617}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.878, "count": 199, "max_ms": 9.949, "p95_ms": 8.574}, "gstreamer_push": {"avg_ms": 1.052, "count": 199, "max_ms": 1.701, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.66, "count": 199, "max_ms": 86.205, "p95_ms": 77.283}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.868, "count": 199, "max_ms": 115.774, "p95_ms": 103.493}, "yolo11_640_fire_11class": {"avg_ms": 60.58, "count": 199, "max_ms": 89.032, "p95_ms": 82.538}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.152, "count": 199, "max_ms": 105.253, "p95_ms": 99.081}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.325, "count": 199, "max_ms": 199.27, "p95_ms": 181.083}, "read_interval": {"avg_ms": 150.934, "count": 198, "max_ms": 207.897, "p95_ms": 188.888}, "triton_wait": {"avg_ms": 74.332, "count": 199, "max_ms": 115.785, "p95_ms": 103.504}, "visualize": {"avg_ms": 5.872, "count": 199, "max_ms": 8.217, "p95_ms": 7.048}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5968.793, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 319, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:54:33+09:00", "last_error_summary": "[h264 @ 0x55c918f007c0] error while decoding MB 75 22, bytestream -5", "last_exit_at": "2026-07-06T19:54:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.853, "last_started_at": "2026-07-06T19:51:49+09:00", "pid": 25030, "process": {"exists": false, "pid": 25030}, "restart_count": 103, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 104}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.545, "sample_count": 1968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 198169, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:54:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T19:54:33+09:00", "last_frame_read_age_sec": 0.19, "last_frame_read_at": "2026-07-06T19:54:33+09:00", "mjpeg": {"age_sec": 0.18, "exists": true, "mtime": "2026-07-06T19:54:33.329107+09:00", "mtime_epoch": 1783335273.3291066, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178877}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:54:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1632.648, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 199, "pushed_frames": 199, "read_frames": 198, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.649, "pushed": 6.649, "read": 6.616}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.878, "count": 199, "max_ms": 9.949, "p95_ms": 8.574}, "gstreamer_push": {"avg_ms": 1.052, "count": 199, "max_ms": 1.701, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.66, "count": 199, "max_ms": 86.205, "p95_ms": 77.283}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.868, "count": 199, "max_ms": 115.774, "p95_ms": 103.493}, "yolo11_640_fire_11class": {"avg_ms": 60.58, "count": 199, "max_ms": 89.032, "p95_ms": 82.538}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.152, "count": 199, "max_ms": 105.253, "p95_ms": 99.081}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.325, "count": 199, "max_ms": 199.27, "p95_ms": 181.083}, "read_interval": {"avg_ms": 150.934, "count": 198, "max_ms": 207.897, "p95_ms": 188.888}, "triton_wait": {"avg_ms": 74.332, "count": 199, "max_ms": 115.785, "p95_ms": 103.504}, "visualize": {"avg_ms": 5.872, "count": 199, "max_ms": 8.217, "p95_ms": 7.048}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6078.301, "limit_mb": 8796093022207.996}, "detail": {"pid": 25137, "restart_count": 104, "start_count": 105}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 319, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:54:33+09:00", "last_error_summary": "[h264 @ 0x55c918f007c0] error while decoding MB 75 22, bytestream -5", "last_exit_at": "2026-07-06T19:54:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:54:36+09:00", "pid": 25137, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.754, "name": "ffmpeg", "pid": 25137, "rss_mb": 138.754, "state": "S (sleeping)", "threads": 106}, "restart_count": 104, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 105}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.495, "sample_count": 1937, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 198169, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:54:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.818, "last_frame_pushed_at": "2026-07-06T19:54:33+09:00", "last_frame_read_age_sec": 4.997, "last_frame_read_at": "2026-07-06T19:54:33+09:00", "mjpeg": {"age_sec": 4.799, "exists": true, "mtime": "2026-07-06T19:54:33.517108+09:00", "mtime_epoch": 1783335273.5171082, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178876}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:54:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1632.648, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 166, "pushed_frames": 166, "read_frames": 165, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.539, "pushed": 5.539, "read": 5.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.896, "count": 165, "max_ms": 9.949, "p95_ms": 8.642}, "gstreamer_push": {"avg_ms": 1.045, "count": 166, "max_ms": 1.701, "p95_ms": 1.183}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.353, "count": 165, "max_ms": 86.205, "p95_ms": 77.157}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.487, "count": 165, "max_ms": 115.774, "p95_ms": 103.493}, "yolo11_640_fire_11class": {"avg_ms": 60.639, "count": 165, "max_ms": 87.612, "p95_ms": 82.257}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.267, "count": 165, "max_ms": 105.253, "p95_ms": 98.019}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.57, "count": 166, "max_ms": 199.27, "p95_ms": 180.303}, "read_interval": {"avg_ms": 152.069, "count": 165, "max_ms": 207.897, "p95_ms": 188.195}, "triton_wait": {"avg_ms": 73.977, "count": 165, "max_ms": 115.785, "p95_ms": 103.504}, "visualize": {"avg_ms": 5.929, "count": 165, "max_ms": 8.217, "p95_ms": 6.98}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1660}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5969.191, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fe1fba7240] error while decoding MB 4 25, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 320, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:57:16+09:00", "last_error_summary": "[h264 @ 0x55fe1fba7240] error while decoding MB 4 25, bytestream -7", "last_exit_at": "2026-07-06T19:54:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.499, "last_started_at": "2026-07-06T19:54:36+09:00", "pid": 25137, "process": {"exists": false, "pid": 25137}, "restart_count": 104, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 105}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.519, "sample_count": 1960, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 199214, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:57:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T19:57:16+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T19:57:16+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T19:57:16.770702+09:00", "mtime_epoch": 1783335436.7707021, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166461}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:57:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.164, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.592, "pushed": 6.592, "read": 6.592}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.05, "count": 197, "max_ms": 11.125, "p95_ms": 9.007}, "gstreamer_push": {"avg_ms": 1.048, "count": 197, "max_ms": 1.603, "p95_ms": 1.197}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.216, "count": 197, "max_ms": 90.8, "p95_ms": 73.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.487, "count": 197, "max_ms": 117.718, "p95_ms": 96.58}, "yolo11_640_fire_11class": {"avg_ms": 60.37, "count": 197, "max_ms": 94.469, "p95_ms": 81.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.483, "count": 197, "max_ms": 105.557, "p95_ms": 92.778}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.372, "count": 197, "max_ms": 197.799, "p95_ms": 176.018}, "read_interval": {"avg_ms": 152.099, "count": 197, "max_ms": 205.966, "p95_ms": 185.24}, "triton_wait": {"avg_ms": 72.993, "count": 197, "max_ms": 117.73, "p95_ms": 98.951}, "visualize": {"avg_ms": 6.019, "count": 197, "max_ms": 8.374, "p95_ms": 7.492}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1660}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5959.613, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:57:16+09:00", "last_error_summary": "[h264 @ 0x55fe1fba7240] error while decoding MB 4 25, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 320, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:57:16+09:00", "last_error_summary": "[h264 @ 0x55fe1fba7240] error while decoding MB 4 25, bytestream -7", "last_exit_at": "2026-07-06T19:57:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.595, "last_started_at": "2026-07-06T19:54:36+09:00", "pid": 25137, "process": {"exists": false, "pid": 25137}, "restart_count": 104, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 105}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.519, "sample_count": 1960, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 199215, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:57:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T19:57:16+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-06T19:57:16+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-06T19:57:16.770702+09:00", "mtime_epoch": 1783335436.7707021, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166461}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:57:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.164, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.605, "pushed": 6.605, "read": 6.571}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.058, "count": 197, "max_ms": 11.125, "p95_ms": 9.007}, "gstreamer_push": {"avg_ms": 1.048, "count": 198, "max_ms": 1.603, "p95_ms": 1.197}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.242, "count": 197, "max_ms": 90.8, "p95_ms": 73.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.478, "count": 197, "max_ms": 117.718, "p95_ms": 96.58}, "yolo11_640_fire_11class": {"avg_ms": 60.367, "count": 197, "max_ms": 94.469, "p95_ms": 81.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.47, "count": 197, "max_ms": 105.557, "p95_ms": 92.778}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.356, "count": 198, "max_ms": 197.799, "p95_ms": 176.018}, "read_interval": {"avg_ms": 152.099, "count": 197, "max_ms": 205.966, "p95_ms": 185.24}, "triton_wait": {"avg_ms": 72.984, "count": 197, "max_ms": 117.73, "p95_ms": 98.951}, "visualize": {"avg_ms": 6.023, "count": 197, "max_ms": 8.374, "p95_ms": 7.492}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1660}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5959.605, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 320, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:57:16+09:00", "last_error_summary": "[h264 @ 0x55fe1fba7240] error while decoding MB 4 25, bytestream -7", "last_exit_at": "2026-07-06T19:57:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.602, "last_started_at": "2026-07-06T19:54:36+09:00", "pid": 25137, "process": {"exists": false, "pid": 25137}, "restart_count": 104, "restart_count_per_hour": 22, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 105}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.519, "sample_count": 1960, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 199215, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:57:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T19:57:16+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-06T19:57:16+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-06T19:57:16.770702+09:00", "mtime_epoch": 1783335436.7707021, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166461}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:57:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.164, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.603, "pushed": 6.603, "read": 6.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.058, "count": 197, "max_ms": 11.125, "p95_ms": 9.007}, "gstreamer_push": {"avg_ms": 1.048, "count": 198, "max_ms": 1.603, "p95_ms": 1.197}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.242, "count": 197, "max_ms": 90.8, "p95_ms": 73.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.478, "count": 197, "max_ms": 117.718, "p95_ms": 96.58}, "yolo11_640_fire_11class": {"avg_ms": 60.367, "count": 197, "max_ms": 94.469, "p95_ms": 81.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.47, "count": 197, "max_ms": 105.557, "p95_ms": 92.778}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.356, "count": 198, "max_ms": 197.799, "p95_ms": 176.018}, "read_interval": {"avg_ms": 152.099, "count": 197, "max_ms": 205.966, "p95_ms": 185.24}, "triton_wait": {"avg_ms": 72.984, "count": 197, "max_ms": 117.73, "p95_ms": 98.951}, "visualize": {"avg_ms": 6.023, "count": 197, "max_ms": 8.374, "p95_ms": 7.492}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1660}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6073.043, "limit_mb": 8796093022207.996}, "detail": {"pid": 25244, "restart_count": 105, "start_count": 106}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 320, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:57:16+09:00", "last_error_summary": "[h264 @ 0x55fe1fba7240] error while decoding MB 4 25, bytestream -7", "last_exit_at": "2026-07-06T19:57:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:57:19+09:00", "pid": 25244, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.422, "name": "ffmpeg", "pid": 25244, "rss_mb": 139.422, "state": "S (sleeping)", "threads": 106}, "restart_count": 105, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 106}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.741, "sample_count": 1927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 199215, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:57:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.969, "last_frame_pushed_at": "2026-07-06T19:57:16+09:00", "last_frame_read_age_sec": 5.111, "last_frame_read_at": "2026-07-06T19:57:16+09:00", "mjpeg": {"age_sec": 4.951, "exists": true, "mtime": "2026-07-06T19:57:16.922704+09:00", "mtime_epoch": 1783335436.9227037, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165962}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:57:21+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.164, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 163, "pushed_frames": 163, "read_frames": 162, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.437, "pushed": 5.437, "read": 5.404}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.159, "count": 162, "max_ms": 11.125, "p95_ms": 9.196}, "gstreamer_push": {"avg_ms": 1.04, "count": 163, "max_ms": 1.449, "p95_ms": 1.184}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.243, "count": 162, "max_ms": 90.8, "p95_ms": 74.456}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.571, "count": 162, "max_ms": 117.718, "p95_ms": 98.939}, "yolo11_640_fire_11class": {"avg_ms": 60.46, "count": 162, "max_ms": 94.469, "p95_ms": 81.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.347, "count": 162, "max_ms": 105.557, "p95_ms": 92.787}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.643, "count": 163, "max_ms": 197.799, "p95_ms": 176.878}, "read_interval": {"avg_ms": 154.368, "count": 162, "max_ms": 205.966, "p95_ms": 186.585}, "triton_wait": {"avg_ms": 73.112, "count": 162, "max_ms": 117.73, "p95_ms": 100.331}, "visualize": {"avg_ms": 6.141, "count": 162, "max_ms": 8.374, "p95_ms": 7.509}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1668}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5976.82, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ee139f4b00] error while decoding MB 44 31, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 96, "h264_decode_error_count": 321, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:59:55+09:00", "last_error_summary": "[h264 @ 0x55ee139f4b00] error while decoding MB 44 31, bytestream -5", "last_exit_at": "2026-07-06T19:57:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.496, "last_started_at": "2026-07-06T19:57:19+09:00", "pid": 25244, "process": {"exists": false, "pid": 25244}, "restart_count": 105, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 106}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.886, "sample_count": 1916, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 200205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:59:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T19:59:55+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T19:59:55+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-06T19:59:55.324592+09:00", "mtime_epoch": 1783335595.3245924, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169669}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:59:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.383, "pushed": 6.383, "read": 6.383}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.905, "count": 191, "max_ms": 10.501, "p95_ms": 8.773}, "gstreamer_push": {"avg_ms": 1.028, "count": 191, "max_ms": 1.418, "p95_ms": 1.217}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.339, "count": 191, "max_ms": 83.755, "p95_ms": 75.072}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.48, "count": 191, "max_ms": 119.292, "p95_ms": 105.84}, "yolo11_640_fire_11class": {"avg_ms": 60.966, "count": 191, "max_ms": 89.035, "p95_ms": 82.264}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.817, "count": 191, "max_ms": 106.186, "p95_ms": 97.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.643, "count": 191, "max_ms": 201.392, "p95_ms": 183.918}, "read_interval": {"avg_ms": 157.222, "count": 191, "max_ms": 209.18, "p95_ms": 192.702}, "triton_wait": {"avg_ms": 75.109, "count": 191, "max_ms": 119.305, "p95_ms": 105.855}, "visualize": {"avg_ms": 5.981, "count": 191, "max_ms": 8.438, "p95_ms": 6.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1668}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5975.934, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T19:59:55+09:00", "last_error_summary": "[h264 @ 0x55ee139f4b00] error while decoding MB 44 31, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 96, "h264_decode_error_count": 321, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:59:55+09:00", "last_error_summary": "[h264 @ 0x55ee139f4b00] error while decoding MB 44 31, bytestream -5", "last_exit_at": "2026-07-06T19:59:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.57, "last_started_at": "2026-07-06T19:57:19+09:00", "pid": 25244, "process": {"exists": false, "pid": 25244}, "restart_count": 105, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 106}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.886, "sample_count": 1916, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 200206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:59:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T19:59:55+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T19:59:55+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-06T19:59:55.324592+09:00", "mtime_epoch": 1783335595.3245924, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169669}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:59:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 192, "pushed_frames": 192, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.4, "pushed": 6.4, "read": 6.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.893, "count": 191, "max_ms": 10.501, "p95_ms": 8.773}, "gstreamer_push": {"avg_ms": 1.028, "count": 192, "max_ms": 1.418, "p95_ms": 1.217}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.328, "count": 191, "max_ms": 83.755, "p95_ms": 75.072}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.392, "count": 191, "max_ms": 119.292, "p95_ms": 105.84}, "yolo11_640_fire_11class": {"avg_ms": 60.844, "count": 191, "max_ms": 89.035, "p95_ms": 82.264}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.743, "count": 191, "max_ms": 106.186, "p95_ms": 97.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.486, "count": 192, "max_ms": 201.392, "p95_ms": 183.918}, "read_interval": {"avg_ms": 157.222, "count": 191, "max_ms": 209.18, "p95_ms": 192.702}, "triton_wait": {"avg_ms": 75.022, "count": 191, "max_ms": 119.305, "p95_ms": 105.855}, "visualize": {"avg_ms": 5.968, "count": 191, "max_ms": 8.438, "p95_ms": 6.925}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1668}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5975.867, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 96, "h264_decode_error_count": 321, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:59:55+09:00", "last_error_summary": "[h264 @ 0x55ee139f4b00] error while decoding MB 44 31, bytestream -5", "last_exit_at": "2026-07-06T19:59:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.575, "last_started_at": "2026-07-06T19:57:19+09:00", "pid": 25244, "process": {"exists": false, "pid": 25244}, "restart_count": 105, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 106}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.886, "sample_count": 1916, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 200206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:59:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T19:59:55+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T19:59:55+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T19:59:55.324592+09:00", "mtime_epoch": 1783335595.3245924, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169669}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:59:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.367, "pushed": 6.367, "read": 6.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.893, "count": 191, "max_ms": 10.501, "p95_ms": 8.773}, "gstreamer_push": {"avg_ms": 1.029, "count": 191, "max_ms": 1.418, "p95_ms": 1.217}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.328, "count": 191, "max_ms": 83.755, "p95_ms": 75.072}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.392, "count": 191, "max_ms": 119.292, "p95_ms": 105.84}, "yolo11_640_fire_11class": {"avg_ms": 60.844, "count": 191, "max_ms": 89.035, "p95_ms": 82.264}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.743, "count": 191, "max_ms": 106.186, "p95_ms": 97.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.436, "count": 191, "max_ms": 201.392, "p95_ms": 183.918}, "read_interval": {"avg_ms": 157.222, "count": 191, "max_ms": 209.18, "p95_ms": 192.702}, "triton_wait": {"avg_ms": 75.022, "count": 191, "max_ms": 119.305, "p95_ms": 105.855}, "visualize": {"avg_ms": 5.968, "count": 191, "max_ms": 8.438, "p95_ms": 6.925}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1668}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6083.262, "limit_mb": 8796093022207.996}, "detail": {"pid": 25351, "restart_count": 106, "start_count": 107}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 96, "h264_decode_error_count": 321, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T19:59:55+09:00", "last_error_summary": "[h264 @ 0x55ee139f4b00] error while decoding MB 44 31, bytestream -5", "last_exit_at": "2026-07-06T19:59:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T19:59:57+09:00", "pid": 25351, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.461, "name": "ffmpeg", "pid": 25351, "rss_mb": 136.461, "state": "S (sleeping)", "threads": 106}, "restart_count": 106, "restart_count_per_hour": 24, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 107}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.886, "sample_count": 1891, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 200206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T19:59:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.227, "last_frame_pushed_at": "2026-07-06T19:59:55+09:00", "last_frame_read_age_sec": 4.347, "last_frame_read_at": "2026-07-06T19:59:55+09:00", "mjpeg": {"age_sec": 4.21, "exists": true, "mtime": "2026-07-06T19:59:55.452594+09:00", "mtime_epoch": 1783335595.4525938, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168650}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T19:59:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 165, "pushed_frames": 165, "read_frames": 164, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.512, "pushed": 5.512, "read": 5.478}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.839, "count": 165, "max_ms": 10.501, "p95_ms": 9.105}, "gstreamer_push": {"avg_ms": 1.033, "count": 165, "max_ms": 1.418, "p95_ms": 1.217}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.304, "count": 165, "max_ms": 83.755, "p95_ms": 75.072}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.469, "count": 165, "max_ms": 119.292, "p95_ms": 105.84}, "yolo11_640_fire_11class": {"avg_ms": 60.399, "count": 165, "max_ms": 89.035, "p95_ms": 82.264}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.618, "count": 165, "max_ms": 106.186, "p95_ms": 97.899}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.94, "count": 165, "max_ms": 201.392, "p95_ms": 183.918}, "read_interval": {"avg_ms": 156.757, "count": 164, "max_ms": 209.18, "p95_ms": 192.137}, "triton_wait": {"avg_ms": 75.095, "count": 165, "max_ms": 119.305, "p95_ms": 105.855}, "visualize": {"avg_ms": 5.925, "count": 165, "max_ms": 8.041, "p95_ms": 6.814}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5976.254, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b370b1d900] error while decoding MB 60 39, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 97, "h264_decode_error_count": 322, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:02:29+09:00", "last_error_summary": "[h264 @ 0x55b370b1d900] error while decoding MB 60 39, bytestream -5", "last_exit_at": "2026-07-06T19:59:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 151.967, "last_started_at": "2026-07-06T19:59:57+09:00", "pid": 25351, "process": {"exists": false, "pid": 25351}, "restart_count": 106, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 107}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 182.351, "sample_count": 1895, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 201160, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:02:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T20:02:29+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T20:02:29+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T20:02:29.586668+09:00", "mtime_epoch": 1783335749.5866685, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171267}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:02:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.938, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.366, "pushed": 6.366, "read": 6.366}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.229, "count": 190, "max_ms": 10.655, "p95_ms": 8.896}, "gstreamer_push": {"avg_ms": 1.009, "count": 190, "max_ms": 1.306, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.35, "count": 190, "max_ms": 84.329, "p95_ms": 72.223}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.89, "count": 190, "max_ms": 117.617, "p95_ms": 97.603}, "yolo11_640_fire_11class": {"avg_ms": 58.415, "count": 190, "max_ms": 85.834, "p95_ms": 77.298}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.288, "count": 190, "max_ms": 104.882, "p95_ms": 92.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.5, "count": 190, "max_ms": 204.546, "p95_ms": 177.361}, "read_interval": {"avg_ms": 157.708, "count": 190, "max_ms": 212.478, "p95_ms": 186.285}, "triton_wait": {"avg_ms": 72.333, "count": 190, "max_ms": 117.63, "p95_ms": 97.826}, "visualize": {"avg_ms": 6.322, "count": 190, "max_ms": 9.03, "p95_ms": 7.583}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5975.82, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:02:29+09:00", "last_error_summary": "[h264 @ 0x55b370b1d900] error while decoding MB 60 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 97, "h264_decode_error_count": 322, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:02:29+09:00", "last_error_summary": "[h264 @ 0x55b370b1d900] error while decoding MB 60 39, bytestream -5", "last_exit_at": "2026-07-06T20:02:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.076, "last_started_at": "2026-07-06T19:59:57+09:00", "pid": 25351, "process": {"exists": false, "pid": 25351}, "restart_count": 106, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 107}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 182.347, "sample_count": 1895, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 201161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:02:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-06T20:02:29+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-06T20:02:29+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-06T20:02:29.586668+09:00", "mtime_epoch": 1783335749.5866685, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171267}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:02:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.938, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.376, "pushed": 6.376, "read": 6.343}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.22, "count": 190, "max_ms": 10.655, "p95_ms": 8.896}, "gstreamer_push": {"avg_ms": 1.009, "count": 191, "max_ms": 1.306, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.282, "count": 190, "max_ms": 84.329, "p95_ms": 71.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.844, "count": 190, "max_ms": 117.617, "p95_ms": 97.603}, "yolo11_640_fire_11class": {"avg_ms": 58.381, "count": 190, "max_ms": 85.834, "p95_ms": 77.298}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.222, "count": 190, "max_ms": 104.882, "p95_ms": 92.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.533, "count": 191, "max_ms": 204.546, "p95_ms": 177.361}, "read_interval": {"avg_ms": 157.708, "count": 190, "max_ms": 212.478, "p95_ms": 186.285}, "triton_wait": {"avg_ms": 72.279, "count": 190, "max_ms": 117.63, "p95_ms": 97.826}, "visualize": {"avg_ms": 6.319, "count": 190, "max_ms": 9.03, "p95_ms": 7.583}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5975.938, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 97, "h264_decode_error_count": 322, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:02:29+09:00", "last_error_summary": "[h264 @ 0x55b370b1d900] error while decoding MB 60 39, bytestream -5", "last_exit_at": "2026-07-06T20:02:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.081, "last_started_at": "2026-07-06T19:59:57+09:00", "pid": 25351, "process": {"exists": false, "pid": 25351}, "restart_count": 106, "restart_count_per_hour": 23, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 107}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 182.347, "sample_count": 1895, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 201161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:02:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T20:02:29+09:00", "last_frame_read_age_sec": 0.166, "last_frame_read_at": "2026-07-06T20:02:29+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-06T20:02:29.586668+09:00", "mtime_epoch": 1783335749.5866685, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171267}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:02:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.938, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.375, "pushed": 6.375, "read": 6.342}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.22, "count": 190, "max_ms": 10.655, "p95_ms": 8.896}, "gstreamer_push": {"avg_ms": 1.009, "count": 191, "max_ms": 1.306, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.282, "count": 190, "max_ms": 84.329, "p95_ms": 71.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.844, "count": 190, "max_ms": 117.617, "p95_ms": 97.603}, "yolo11_640_fire_11class": {"avg_ms": 58.381, "count": 190, "max_ms": 85.834, "p95_ms": 77.298}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.222, "count": 190, "max_ms": 104.882, "p95_ms": 92.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.533, "count": 191, "max_ms": 204.546, "p95_ms": 177.361}, "read_interval": {"avg_ms": 157.708, "count": 190, "max_ms": 212.478, "p95_ms": 186.285}, "triton_wait": {"avg_ms": 72.279, "count": 190, "max_ms": 117.63, "p95_ms": 97.826}, "visualize": {"avg_ms": 6.319, "count": 190, "max_ms": 9.03, "p95_ms": 7.583}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6083.156, "limit_mb": 8796093022207.996}, "detail": {"pid": 25458, "restart_count": 107, "start_count": 108}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 97, "h264_decode_error_count": 322, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:02:29+09:00", "last_error_summary": "[h264 @ 0x55b370b1d900] error while decoding MB 60 39, bytestream -5", "last_exit_at": "2026-07-06T20:02:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:02:32+09:00", "pid": 25458, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.574, "name": "ffmpeg", "pid": 25458, "rss_mb": 138.574, "state": "S (sleeping)", "threads": 106}, "restart_count": 107, "restart_count_per_hour": 24, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 108}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 182.598, "sample_count": 1863, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 201161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:02:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.741, "last_frame_pushed_at": "2026-07-06T20:02:29+09:00", "last_frame_read_age_sec": 4.898, "last_frame_read_at": "2026-07-06T20:02:29+09:00", "mjpeg": {"age_sec": 4.724, "exists": true, "mtime": "2026-07-06T20:02:29.750671+09:00", "mtime_epoch": 1783335749.750671, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170771}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:02:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1633.938, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.384, "pushed": 5.384, "read": 5.351}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.215, "count": 161, "max_ms": 10.655, "p95_ms": 8.873}, "gstreamer_push": {"avg_ms": 1.008, "count": 161, "max_ms": 1.238, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.823, "count": 161, "max_ms": 80.842, "p95_ms": 70.598}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.601, "count": 161, "max_ms": 115.133, "p95_ms": 97.096}, "yolo11_640_fire_11class": {"avg_ms": 58.153, "count": 161, "max_ms": 85.834, "p95_ms": 76.792}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.968, "count": 161, "max_ms": 98.925, "p95_ms": 89.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.044, "count": 161, "max_ms": 200.141, "p95_ms": 176.453}, "read_interval": {"avg_ms": 157.202, "count": 160, "max_ms": 209.984, "p95_ms": 184.206}, "triton_wait": {"avg_ms": 71.956, "count": 161, "max_ms": 115.145, "p95_ms": 97.11}, "visualize": {"avg_ms": 6.327, "count": 161, "max_ms": 9.03, "p95_ms": 7.431}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1682}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5980.945, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558d42624240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d42624240] error while decoding MB 0 39, bytestream 8111 [h264 @ 0x558d4274a2c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d4274a2c0] error while decoding MB 0 26, bytestream 23949 [h264 @ 0x558d4264f540] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558d4264f540] error while decoding MB 26 24, bytestream 5763 [in#0/rtsp @ 0x558d425e0e00] Error during demuxing: Connection t", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 98, "h264_decode_error_count": 326, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:04:47+09:00", "last_error_summary": "[h264 @ 0x558d42624240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d42624240] error while decoding MB 0 39, bytestream 8111 [h264 @ 0x558d4274a2c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d4274a2c0] error while decoding MB 0 26, bytestream 23949 [h264 @ 0x558d4264f540] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558d4264f540] error while decoding MB 26 24, bytestream 5763 [in#0/rtsp @ 0x558d425e0e00] Error during demuxing: Connection t", "last_exit_at": "2026-07-06T20:02:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.555, "last_started_at": "2026-07-06T20:02:32+09:00", "pid": 25458, "process": {"exists": false, "pid": 25458}, "restart_count": 107, "restart_count_per_hour": 23, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 108}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.746, "sample_count": 1721, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 201872, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:04:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.041, "last_frame_pushed_at": "2026-07-06T20:04:46+09:00", "last_frame_read_age_sec": 0.035, "last_frame_read_at": "2026-07-06T20:04:46+09:00", "mjpeg": {"age_sec": 0.024, "exists": true, "mtime": "2026-07-06T20:04:47.004665+09:00", "mtime_epoch": 1783335887.0046647, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176811}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:04:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1634.453, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 60, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.0, "pushed": 2.0, "read": 2.0}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.026, "count": 59, "max_ms": 9.711, "p95_ms": 8.375}, "gstreamer_push": {"avg_ms": 0.996, "count": 60, "max_ms": 1.175, "p95_ms": 1.153}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.002, "count": 59, "max_ms": 77.867, "p95_ms": 75.174}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.009, "count": 59, "max_ms": 111.816, "p95_ms": 100.264}, "yolo11_640_fire_11class": {"avg_ms": 58.3, "count": 59, "max_ms": 87.267, "p95_ms": 79.599}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.524, "count": 59, "max_ms": 100.974, "p95_ms": 93.375}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.932, "count": 60, "max_ms": 194.556, "p95_ms": 180.81}, "read_interval": {"avg_ms": 501.729, "count": 60, "max_ms": 10633.417, "p95_ms": 191.572}, "triton_wait": {"avg_ms": 73.334, "count": 59, "max_ms": 111.831, "p95_ms": 100.985}, "visualize": {"avg_ms": 6.359, "count": 59, "max_ms": 9.765, "p95_ms": 7.204}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1682}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5980.16, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:04:47+09:00", "last_error_summary": "[h264 @ 0x558d42624240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d42624240] error while decoding MB 0 39, bytestream 8111 [h264 @ 0x558d4274a2c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d4274a2c0] error while decoding MB 0 26, bytestream 23949 [h264 @ 0x558d4264f540] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558d4264f540] error while decoding MB 26 24, bytestream 5763 [in#0/rtsp @ 0x558d425e0e00] Error during demuxing: Connection t", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 98, "h264_decode_error_count": 326, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:04:47+09:00", "last_error_summary": "[h264 @ 0x558d42624240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d42624240] error while decoding MB 0 39, bytestream 8111 [h264 @ 0x558d4274a2c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d4274a2c0] error while decoding MB 0 26, bytestream 23949 [h264 @ 0x558d4264f540] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558d4264f540] error while decoding MB 26 24, bytestream 5763 [in#0/rtsp @ 0x558d425e0e00] Error during demuxing: Connection t", "last_exit_at": "2026-07-06T20:04:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.694, "last_started_at": "2026-07-06T20:02:32+09:00", "pid": 25458, "process": {"exists": false, "pid": 25458}, "restart_count": 107, "restart_count_per_hour": 23, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 108}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.746, "sample_count": 1721, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 201873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:04:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-06T20:04:47+09:00", "last_frame_read_age_sec": 0.174, "last_frame_read_at": "2026-07-06T20:04:46+09:00", "mjpeg": {"age_sec": 0.163, "exists": true, "mtime": "2026-07-06T20:04:47.004665+09:00", "mtime_epoch": 1783335887.0046647, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176811}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:04:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1634.453, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.001, "pushed": 2.001, "read": 1.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.025, "count": 59, "max_ms": 9.711, "p95_ms": 8.375}, "gstreamer_push": {"avg_ms": 0.994, "count": 60, "max_ms": 1.175, "p95_ms": 1.122}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.922, "count": 59, "max_ms": 77.867, "p95_ms": 75.174}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.472, "count": 59, "max_ms": 111.816, "p95_ms": 100.264}, "yolo11_640_fire_11class": {"avg_ms": 58.411, "count": 59, "max_ms": 87.267, "p95_ms": 79.599}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.599, "count": 59, "max_ms": 100.974, "p95_ms": 93.375}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.314, "count": 60, "max_ms": 194.556, "p95_ms": 180.81}, "read_interval": {"avg_ms": 507.626, "count": 59, "max_ms": 10633.417, "p95_ms": 204.716}, "triton_wait": {"avg_ms": 73.797, "count": 59, "max_ms": 111.831, "p95_ms": 100.985}, "visualize": {"avg_ms": 6.359, "count": 59, "max_ms": 9.765, "p95_ms": 7.204}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1682}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5980.152, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 98, "h264_decode_error_count": 326, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:04:47+09:00", "last_error_summary": "[h264 @ 0x558d42624240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d42624240] error while decoding MB 0 39, bytestream 8111 [h264 @ 0x558d4274a2c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d4274a2c0] error while decoding MB 0 26, bytestream 23949 [h264 @ 0x558d4264f540] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558d4264f540] error while decoding MB 26 24, bytestream 5763 [in#0/rtsp @ 0x558d425e0e00] Error during demuxing: Connection t", "last_exit_at": "2026-07-06T20:04:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.698, "last_started_at": "2026-07-06T20:02:32+09:00", "pid": 25458, "process": {"exists": false, "pid": 25458}, "restart_count": 107, "restart_count_per_hour": 23, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 108}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.746, "sample_count": 1721, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 201873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:04:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T20:04:47+09:00", "last_frame_read_age_sec": 0.178, "last_frame_read_at": "2026-07-06T20:04:46+09:00", "mjpeg": {"age_sec": 0.167, "exists": true, "mtime": "2026-07-06T20:04:47.004665+09:00", "mtime_epoch": 1783335887.0046647, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176811}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:04:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1634.453, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.001, "pushed": 2.001, "read": 1.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.025, "count": 59, "max_ms": 9.711, "p95_ms": 8.375}, "gstreamer_push": {"avg_ms": 0.994, "count": 60, "max_ms": 1.175, "p95_ms": 1.122}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.922, "count": 59, "max_ms": 77.867, "p95_ms": 75.174}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.472, "count": 59, "max_ms": 111.816, "p95_ms": 100.264}, "yolo11_640_fire_11class": {"avg_ms": 58.411, "count": 59, "max_ms": 87.267, "p95_ms": 79.599}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.599, "count": 59, "max_ms": 100.974, "p95_ms": 93.375}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.314, "count": 60, "max_ms": 194.556, "p95_ms": 180.81}, "read_interval": {"avg_ms": 507.626, "count": 59, "max_ms": 10633.417, "p95_ms": 204.716}, "triton_wait": {"avg_ms": 73.797, "count": 59, "max_ms": 111.831, "p95_ms": 100.985}, "visualize": {"avg_ms": 6.359, "count": 59, "max_ms": 9.765, "p95_ms": 7.204}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1682}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6087.48, "limit_mb": 8796093022207.996}, "detail": {"pid": 25565, "restart_count": 108, "start_count": 109}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 98, "h264_decode_error_count": 326, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:04:47+09:00", "last_error_summary": "[h264 @ 0x558d42624240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d42624240] error while decoding MB 0 39, bytestream 8111 [h264 @ 0x558d4274a2c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558d4274a2c0] error while decoding MB 0 26, bytestream 23949 [h264 @ 0x558d4264f540] cabac decode of qscale diff failed at 26 24 [h264 @ 0x558d4264f540] error while decoding MB 26 24, bytestream 5763 [in#0/rtsp @ 0x558d425e0e00] Error during demuxing: Connection t", "last_exit_at": "2026-07-06T20:04:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:04:49+09:00", "pid": 25565, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.355, "name": "ffmpeg", "pid": 25565, "rss_mb": 139.355, "state": "S (sleeping)", "threads": 106}, "restart_count": 108, "restart_count_per_hour": 24, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 109}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.192, "sample_count": 1691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 201873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:04:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.622, "last_frame_pushed_at": "2026-07-06T20:04:47+09:00", "last_frame_read_age_sec": 4.792, "last_frame_read_at": "2026-07-06T20:04:46+09:00", "mjpeg": {"age_sec": 4.605, "exists": true, "mtime": "2026-07-06T20:04:47.180667+09:00", "mtime_epoch": 1783335887.1806674, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176811}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:04:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1634.453, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 33, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.649, "pushed": 1.649, "read": 1.649}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.158, "count": 33, "max_ms": 9.711, "p95_ms": 9.043}, "gstreamer_push": {"avg_ms": 1.016, "count": 33, "max_ms": 1.175, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.331, "count": 33, "max_ms": 77.867, "p95_ms": 76.437}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.784, "count": 33, "max_ms": 111.816, "p95_ms": 100.264}, "yolo11_640_fire_11class": {"avg_ms": 61.578, "count": 33, "max_ms": 87.267, "p95_ms": 81.769}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.19, "count": 33, "max_ms": 100.974, "p95_ms": 100.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.449, "count": 33, "max_ms": 194.556, "p95_ms": 182.883}, "read_interval": {"avg_ms": 783.293, "count": 33, "max_ms": 10633.417, "p95_ms": 10134.985}, "triton_wait": {"avg_ms": 76.044, "count": 33, "max_ms": 111.831, "p95_ms": 100.985}, "visualize": {"avg_ms": 6.476, "count": 33, "max_ms": 9.765, "p95_ms": 7.204}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 20.008, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5982.422, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5556e385f880] left block unavailable for requested intra mode [h264 @ 0x5556e385f880] error while decoding MB 0 25, bytestream 3190 [h264 @ 0x5556e37cd380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5556e37cd380] error while decoding MB 0 34, bytestream 630 [in#0/rtsp @ 0x5556e3705e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556e3705e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 99, "h264_decode_error_count": 328, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:06:50+09:00", "last_error_summary": "[h264 @ 0x5556e385f880] left block unavailable for requested intra mode [h264 @ 0x5556e385f880] error while decoding MB 0 25, bytestream 3190 [h264 @ 0x5556e37cd380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5556e37cd380] error while decoding MB 0 34, bytestream 630 [in#0/rtsp @ 0x5556e3705e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556e3705e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:04:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.587, "last_started_at": "2026-07-06T20:04:49+09:00", "pid": 25565, "process": {"exists": false, "pid": 25565}, "restart_count": 108, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 109}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.701, "sample_count": 1674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 202588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:06:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-06T20:06:50+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T20:06:50+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T20:06:50.330546+09:00", "mtime_epoch": 1783336010.330546, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184074}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:06:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1634.711, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.314, "pushed": 4.314, "read": 4.314}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.943, "count": 129, "max_ms": 9.79, "p95_ms": 8.392}, "gstreamer_push": {"avg_ms": 1.013, "count": 129, "max_ms": 1.26, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.744, "count": 129, "max_ms": 89.827, "p95_ms": 74.747}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.848, "count": 129, "max_ms": 113.529, "p95_ms": 100.783}, "yolo11_640_fire_11class": {"avg_ms": 59.037, "count": 129, "max_ms": 90.934, "p95_ms": 76.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.783, "count": 129, "max_ms": 105.654, "p95_ms": 91.303}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.068, "count": 129, "max_ms": 194.432, "p95_ms": 179.765}, "read_interval": {"avg_ms": 232.308, "count": 129, "max_ms": 9813.485, "p95_ms": 192.207}, "triton_wait": {"avg_ms": 74.085, "count": 129, "max_ms": 113.543, "p95_ms": 100.81}, "visualize": {"avg_ms": 6.144, "count": 129, "max_ms": 8.093, "p95_ms": 7.089}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5981.41, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:06:50+09:00", "last_error_summary": "[h264 @ 0x5556e385f880] left block unavailable for requested intra mode [h264 @ 0x5556e385f880] error while decoding MB 0 25, bytestream 3190 [h264 @ 0x5556e37cd380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5556e37cd380] error while decoding MB 0 34, bytestream 630 [in#0/rtsp @ 0x5556e3705e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556e3705e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 99, "h264_decode_error_count": 328, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:06:50+09:00", "last_error_summary": "[h264 @ 0x5556e385f880] left block unavailable for requested intra mode [h264 @ 0x5556e385f880] error while decoding MB 0 25, bytestream 3190 [h264 @ 0x5556e37cd380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5556e37cd380] error while decoding MB 0 34, bytestream 630 [in#0/rtsp @ 0x5556e3705e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556e3705e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:06:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.7, "last_started_at": "2026-07-06T20:04:49+09:00", "pid": 25565, "process": {"exists": false, "pid": 25565}, "restart_count": 108, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 109}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.701, "sample_count": 1675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 202589, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:06:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T20:06:50+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-06T20:06:50+09:00", "mjpeg": {"age_sec": 0.154, "exists": true, "mtime": "2026-07-06T20:06:50.330546+09:00", "mtime_epoch": 1783336010.330546, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184074}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:06:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1634.711, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.315, "pushed": 4.315, "read": 4.281}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.971, "count": 129, "max_ms": 9.79, "p95_ms": 8.583}, "gstreamer_push": {"avg_ms": 1.014, "count": 129, "max_ms": 1.26, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.655, "count": 129, "max_ms": 89.827, "p95_ms": 74.747}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.919, "count": 129, "max_ms": 113.529, "p95_ms": 100.783}, "yolo11_640_fire_11class": {"avg_ms": 59.072, "count": 129, "max_ms": 90.934, "p95_ms": 76.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.815, "count": 129, "max_ms": 105.654, "p95_ms": 91.303}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.376, "count": 129, "max_ms": 194.432, "p95_ms": 179.765}, "read_interval": {"avg_ms": 233.118, "count": 128, "max_ms": 9813.485, "p95_ms": 192.207}, "triton_wait": {"avg_ms": 74.157, "count": 129, "max_ms": 113.543, "p95_ms": 100.81}, "visualize": {"avg_ms": 6.166, "count": 129, "max_ms": 8.093, "p95_ms": 7.11}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5981.434, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 99, "h264_decode_error_count": 328, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:06:50+09:00", "last_error_summary": "[h264 @ 0x5556e385f880] left block unavailable for requested intra mode [h264 @ 0x5556e385f880] error while decoding MB 0 25, bytestream 3190 [h264 @ 0x5556e37cd380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5556e37cd380] error while decoding MB 0 34, bytestream 630 [in#0/rtsp @ 0x5556e3705e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556e3705e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:06:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.703, "last_started_at": "2026-07-06T20:04:49+09:00", "pid": 25565, "process": {"exists": false, "pid": 25565}, "restart_count": 108, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 109}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.701, "sample_count": 1675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 202589, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:06:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T20:06:50+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-06T20:06:50+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-06T20:06:50.330546+09:00", "mtime_epoch": 1783336010.330546, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184074}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:06:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1634.711, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.314, "pushed": 4.314, "read": 4.281}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.971, "count": 129, "max_ms": 9.79, "p95_ms": 8.583}, "gstreamer_push": {"avg_ms": 1.014, "count": 129, "max_ms": 1.26, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.655, "count": 129, "max_ms": 89.827, "p95_ms": 74.747}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.919, "count": 129, "max_ms": 113.529, "p95_ms": 100.783}, "yolo11_640_fire_11class": {"avg_ms": 59.072, "count": 129, "max_ms": 90.934, "p95_ms": 76.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.815, "count": 129, "max_ms": 105.654, "p95_ms": 91.303}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.376, "count": 129, "max_ms": 194.432, "p95_ms": 179.765}, "read_interval": {"avg_ms": 233.118, "count": 128, "max_ms": 9813.485, "p95_ms": 192.207}, "triton_wait": {"avg_ms": 74.157, "count": 129, "max_ms": 113.543, "p95_ms": 100.81}, "visualize": {"avg_ms": 6.166, "count": 129, "max_ms": 8.093, "p95_ms": 7.11}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6087.074, "limit_mb": 8796093022207.996}, "detail": {"pid": 25672, "restart_count": 109, "start_count": 110}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 99, "h264_decode_error_count": 328, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:06:50+09:00", "last_error_summary": "[h264 @ 0x5556e385f880] left block unavailable for requested intra mode [h264 @ 0x5556e385f880] error while decoding MB 0 25, bytestream 3190 [h264 @ 0x5556e37cd380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5556e37cd380] error while decoding MB 0 34, bytestream 630 [in#0/rtsp @ 0x5556e3705e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556e3705e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:06:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:06:52+09:00", "pid": 25672, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.34, "name": "ffmpeg", "pid": 25672, "rss_mb": 139.34, "state": "S (sleeping)", "threads": 106}, "restart_count": 109, "restart_count_per_hour": 24, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 110}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.747, "sample_count": 1646, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 202589, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:06:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.506, "last_frame_pushed_at": "2026-07-06T20:06:50+09:00", "last_frame_read_age_sec": 4.667, "last_frame_read_at": "2026-07-06T20:06:50+09:00", "mjpeg": {"age_sec": 4.49, "exists": true, "mtime": "2026-07-06T20:06:50.494549+09:00", "mtime_epoch": 1783336010.4945486, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184742}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:06:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1634.711, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.34, "pushed": 3.34, "read": 3.306}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.99, "count": 99, "max_ms": 9.79, "p95_ms": 8.811}, "gstreamer_push": {"avg_ms": 1.014, "count": 100, "max_ms": 1.26, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.952, "count": 99, "max_ms": 89.827, "p95_ms": 74.747}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.799, "count": 99, "max_ms": 113.529, "p95_ms": 101.751}, "yolo11_640_fire_11class": {"avg_ms": 59.551, "count": 99, "max_ms": 90.934, "p95_ms": 76.496}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.618, "count": 99, "max_ms": 105.654, "p95_ms": 91.665}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.189, "count": 100, "max_ms": 194.432, "p95_ms": 179.765}, "read_interval": {"avg_ms": 257.027, "count": 99, "max_ms": 9813.485, "p95_ms": 195.715}, "triton_wait": {"avg_ms": 75.09, "count": 99, "max_ms": 113.543, "p95_ms": 101.763}, "visualize": {"avg_ms": 6.188, "count": 99, "max_ms": 8.093, "p95_ms": 7.719}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1696}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5973.004, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560d7c62d6c0] error while decoding MB 15 31, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 329, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:09:22+09:00", "last_error_summary": "[h264 @ 0x560d7c62d6c0] error while decoding MB 15 31, bytestream -9", "last_exit_at": "2026-07-06T20:06:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.757, "last_started_at": "2026-07-06T20:06:52+09:00", "pid": 25672, "process": {"exists": false, "pid": 25672}, "restart_count": 109, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 110}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.897, "sample_count": 1696, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 203536, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:09:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-06T20:09:22+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-06T20:09:22+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T20:09:22.700960+09:00", "mtime_epoch": 1783336162.70096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:09:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1635.227, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.71, "pushed": 6.71, "read": 6.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.851, "count": 200, "max_ms": 10.715, "p95_ms": 8.403}, "gstreamer_push": {"avg_ms": 1.052, "count": 201, "max_ms": 1.641, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.361, "count": 200, "max_ms": 82.952, "p95_ms": 74.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.927, "count": 200, "max_ms": 104.485, "p95_ms": 98.915}, "yolo11_640_fire_11class": {"avg_ms": 60.408, "count": 200, "max_ms": 85.806, "p95_ms": 76.739}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.69, "count": 200, "max_ms": 101.725, "p95_ms": 92.97}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.049, "count": 201, "max_ms": 181.712, "p95_ms": 176.039}, "read_interval": {"avg_ms": 149.518, "count": 201, "max_ms": 189.16, "p95_ms": 183.355}, "triton_wait": {"avg_ms": 74.242, "count": 200, "max_ms": 104.499, "p95_ms": 98.927}, "visualize": {"avg_ms": 5.822, "count": 200, "max_ms": 10.732, "p95_ms": 6.908}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1696}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5971.988, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:09:22+09:00", "last_error_summary": "[h264 @ 0x560d7c62d6c0] error while decoding MB 15 31, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 329, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:09:22+09:00", "last_error_summary": "[h264 @ 0x560d7c62d6c0] error while decoding MB 15 31, bytestream -9", "last_exit_at": "2026-07-06T20:09:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.839, "last_started_at": "2026-07-06T20:06:52+09:00", "pid": 25672, "process": {"exists": false, "pid": 25672}, "restart_count": 109, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 110}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.897, "sample_count": 1697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 203537, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:09:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T20:09:22+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-06T20:09:22+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T20:09:22.700960+09:00", "mtime_epoch": 1783336162.70096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:09:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1635.227, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.733, "pushed": 6.733, "read": 6.7}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.849, "count": 201, "max_ms": 10.715, "p95_ms": 8.403}, "gstreamer_push": {"avg_ms": 1.053, "count": 201, "max_ms": 1.641, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.374, "count": 201, "max_ms": 82.952, "p95_ms": 74.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.891, "count": 201, "max_ms": 104.485, "p95_ms": 98.915}, "yolo11_640_fire_11class": {"avg_ms": 60.399, "count": 201, "max_ms": 85.806, "p95_ms": 76.739}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.666, "count": 201, "max_ms": 101.725, "p95_ms": 92.97}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.935, "count": 201, "max_ms": 181.712, "p95_ms": 176.039}, "read_interval": {"avg_ms": 149.511, "count": 200, "max_ms": 189.16, "p95_ms": 183.355}, "triton_wait": {"avg_ms": 74.205, "count": 201, "max_ms": 104.499, "p95_ms": 98.927}, "visualize": {"avg_ms": 5.818, "count": 201, "max_ms": 10.732, "p95_ms": 6.908}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1696}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5971.98, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 329, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:09:22+09:00", "last_error_summary": "[h264 @ 0x560d7c62d6c0] error while decoding MB 15 31, bytestream -9", "last_exit_at": "2026-07-06T20:09:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.844, "last_started_at": "2026-07-06T20:06:52+09:00", "pid": 25672, "process": {"exists": false, "pid": 25672}, "restart_count": 109, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 110}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.897, "sample_count": 1697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 203537, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:09:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T20:09:22+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-06T20:09:22+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-06T20:09:22.700960+09:00", "mtime_epoch": 1783336162.70096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:09:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1635.227, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.732, "pushed": 6.732, "read": 6.699}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.849, "count": 201, "max_ms": 10.715, "p95_ms": 8.403}, "gstreamer_push": {"avg_ms": 1.053, "count": 201, "max_ms": 1.641, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.374, "count": 201, "max_ms": 82.952, "p95_ms": 74.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.891, "count": 201, "max_ms": 104.485, "p95_ms": 98.915}, "yolo11_640_fire_11class": {"avg_ms": 60.399, "count": 201, "max_ms": 85.806, "p95_ms": 76.739}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.666, "count": 201, "max_ms": 101.725, "p95_ms": 92.97}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.935, "count": 201, "max_ms": 181.712, "p95_ms": 176.039}, "read_interval": {"avg_ms": 149.511, "count": 200, "max_ms": 189.16, "p95_ms": 183.355}, "triton_wait": {"avg_ms": 74.205, "count": 201, "max_ms": 104.499, "p95_ms": 98.927}, "visualize": {"avg_ms": 5.818, "count": 201, "max_ms": 10.732, "p95_ms": 6.908}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.857, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1696}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6092.41, "limit_mb": 8796093022207.996}, "detail": {"pid": 25779, "restart_count": 110, "start_count": 111}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 329, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:09:22+09:00", "last_error_summary": "[h264 @ 0x560d7c62d6c0] error while decoding MB 15 31, bytestream -9", "last_exit_at": "2026-07-06T20:09:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:09:24+09:00", "pid": 25779, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.863, "name": "ffmpeg", "pid": 25779, "rss_mb": 139.863, "state": "S (sleeping)", "threads": 106}, "restart_count": 110, "restart_count_per_hour": 24, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 111}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.897, "sample_count": 1697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 203537, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:09:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.181, "last_frame_pushed_at": "2026-07-06T20:09:22+09:00", "last_frame_read_age_sec": 4.304, "last_frame_read_at": "2026-07-06T20:09:22+09:00", "mjpeg": {"age_sec": 4.169, "exists": true, "mtime": "2026-07-06T20:09:22.828962+09:00", "mtime_epoch": 1783336162.828962, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172787}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:09:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1635.227, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.818, "pushed": 5.818, "read": 5.784}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.854, "count": 174, "max_ms": 10.715, "p95_ms": 8.931}, "gstreamer_push": {"avg_ms": 1.052, "count": 174, "max_ms": 1.641, "p95_ms": 1.19}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.147, "count": 174, "max_ms": 82.952, "p95_ms": 73.73}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.506, "count": 174, "max_ms": 104.485, "p95_ms": 98.182}, "yolo11_640_fire_11class": {"avg_ms": 60.293, "count": 174, "max_ms": 85.806, "p95_ms": 76.423}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.662, "count": 174, "max_ms": 101.725, "p95_ms": 92.514}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.286, "count": 174, "max_ms": 181.576, "p95_ms": 176.039}, "read_interval": {"avg_ms": 148.861, "count": 173, "max_ms": 188.807, "p95_ms": 183.355}, "triton_wait": {"avg_ms": 73.867, "count": 174, "max_ms": 104.499, "p95_ms": 98.194}, "visualize": {"avg_ms": 5.818, "count": 174, "max_ms": 10.732, "p95_ms": 6.908}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5981.258, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556c5c8cce80] error while decoding MB 54 18, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 101, "h264_decode_error_count": 330, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:12:21+09:00", "last_error_summary": "[h264 @ 0x556c5c8cce80] error while decoding MB 54 18, bytestream -13", "last_exit_at": "2026-07-06T20:09:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 176.899, "last_started_at": "2026-07-06T20:09:24+09:00", "pid": 25779, "process": {"exists": false, "pid": 25779}, "restart_count": 110, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 111}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.197, "sample_count": 1997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 204760, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:12:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.052, "last_frame_pushed_at": "2026-07-06T20:12:21+09:00", "last_frame_read_age_sec": 0.043, "last_frame_read_at": "2026-07-06T20:12:21+09:00", "mjpeg": {"age_sec": 0.033, "exists": true, "mtime": "2026-07-06T20:12:21.863893+09:00", "mtime_epoch": 1783336341.8638928, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170001}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:12:21+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1635.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 210, "pushed_frames": 210, "read_frames": 210, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.004, "pushed": 7.004, "read": 7.004}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.529, "count": 209, "max_ms": 9.637, "p95_ms": 7.715}, "gstreamer_push": {"avg_ms": 1.092, "count": 210, "max_ms": 1.361, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.93, "count": 209, "max_ms": 81.78, "p95_ms": 75.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.736, "count": 209, "max_ms": 117.237, "p95_ms": 100.998}, "yolo11_640_fire_11class": {"avg_ms": 59.748, "count": 209, "max_ms": 83.686, "p95_ms": 78.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.126, "count": 209, "max_ms": 100.411, "p95_ms": 93.883}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.813, "count": 210, "max_ms": 186.317, "p95_ms": 166.664}, "read_interval": {"avg_ms": 143.149, "count": 210, "max_ms": 194.906, "p95_ms": 176.13}, "triton_wait": {"avg_ms": 74.958, "count": 209, "max_ms": 117.249, "p95_ms": 101.021}, "visualize": {"avg_ms": 5.401, "count": 209, "max_ms": 8.161, "p95_ms": 6.578}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5980.273, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:12:21+09:00", "last_error_summary": "[h264 @ 0x556c5c8cce80] error while decoding MB 54 18, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 101, "h264_decode_error_count": 330, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:12:21+09:00", "last_error_summary": "[h264 @ 0x556c5c8cce80] error while decoding MB 54 18, bytestream -13", "last_exit_at": "2026-07-06T20:12:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 176.985, "last_started_at": "2026-07-06T20:09:24+09:00", "pid": 25779, "process": {"exists": false, "pid": 25779}, "restart_count": 110, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 111}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.197, "sample_count": 1998, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 204761, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:12:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T20:12:21+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T20:12:21+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-06T20:12:21.863893+09:00", "mtime_epoch": 1783336341.8638928, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170001}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:12:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1635.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 210, "pushed_frames": 210, "read_frames": 209, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.015, "pushed": 7.015, "read": 6.982}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.528, "count": 210, "max_ms": 9.637, "p95_ms": 7.715}, "gstreamer_push": {"avg_ms": 1.093, "count": 210, "max_ms": 1.361, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.907, "count": 210, "max_ms": 81.78, "p95_ms": 75.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.7, "count": 210, "max_ms": 117.237, "p95_ms": 100.998}, "yolo11_640_fire_11class": {"avg_ms": 59.7, "count": 210, "max_ms": 83.686, "p95_ms": 78.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.107, "count": 210, "max_ms": 100.411, "p95_ms": 93.883}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.811, "count": 210, "max_ms": 186.317, "p95_ms": 166.664}, "read_interval": {"avg_ms": 143.21, "count": 209, "max_ms": 194.906, "p95_ms": 176.13}, "triton_wait": {"avg_ms": 74.92, "count": 210, "max_ms": 117.249, "p95_ms": 101.021}, "visualize": {"avg_ms": 5.401, "count": 210, "max_ms": 8.161, "p95_ms": 6.578}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5980.16, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 101, "h264_decode_error_count": 330, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:12:21+09:00", "last_error_summary": "[h264 @ 0x556c5c8cce80] error while decoding MB 54 18, bytestream -13", "last_exit_at": "2026-07-06T20:12:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 176.99, "last_started_at": "2026-07-06T20:09:24+09:00", "pid": 25779, "process": {"exists": false, "pid": 25779}, "restart_count": 110, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 111}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.197, "sample_count": 1998, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 204761, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:12:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T20:12:21+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T20:12:21+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-06T20:12:21.863893+09:00", "mtime_epoch": 1783336341.8638928, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170001}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:12:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1635.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 210, "pushed_frames": 210, "read_frames": 209, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.014, "pushed": 7.014, "read": 6.98}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.528, "count": 210, "max_ms": 9.637, "p95_ms": 7.715}, "gstreamer_push": {"avg_ms": 1.093, "count": 210, "max_ms": 1.361, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.907, "count": 210, "max_ms": 81.78, "p95_ms": 75.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.7, "count": 210, "max_ms": 117.237, "p95_ms": 100.998}, "yolo11_640_fire_11class": {"avg_ms": 59.7, "count": 210, "max_ms": 83.686, "p95_ms": 78.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.107, "count": 210, "max_ms": 100.411, "p95_ms": 93.883}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.811, "count": 210, "max_ms": 186.317, "p95_ms": 166.664}, "read_interval": {"avg_ms": 143.21, "count": 209, "max_ms": 194.906, "p95_ms": 176.13}, "triton_wait": {"avg_ms": 74.92, "count": 210, "max_ms": 117.249, "p95_ms": 101.021}, "visualize": {"avg_ms": 5.401, "count": 210, "max_ms": 8.161, "p95_ms": 6.578}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6086.336, "limit_mb": 8796093022207.996}, "detail": {"pid": 25886, "restart_count": 111, "start_count": 112}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 101, "h264_decode_error_count": 330, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:12:21+09:00", "last_error_summary": "[h264 @ 0x556c5c8cce80] error while decoding MB 54 18, bytestream -13", "last_exit_at": "2026-07-06T20:12:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:12:24+09:00", "pid": 25886, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.105, "name": "ffmpeg", "pid": 25886, "rss_mb": 139.105, "state": "S (sleeping)", "threads": 106}, "restart_count": 111, "restart_count_per_hour": 24, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 112}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.011, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 204761, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:12:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.593, "last_frame_pushed_at": "2026-07-06T20:12:21+09:00", "last_frame_read_age_sec": 4.716, "last_frame_read_at": "2026-07-06T20:12:21+09:00", "mjpeg": {"age_sec": 4.574, "exists": true, "mtime": "2026-07-06T20:12:21.995895+09:00", "mtime_epoch": 1783336341.995895, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:12:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1635.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.935, "pushed": 5.935, "read": 5.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.57, "count": 177, "max_ms": 9.637, "p95_ms": 7.907}, "gstreamer_push": {"avg_ms": 1.093, "count": 178, "max_ms": 1.361, "p95_ms": 1.253}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.3, "count": 177, "max_ms": 79.85, "p95_ms": 74.403}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.149, "count": 177, "max_ms": 117.237, "p95_ms": 100.095}, "yolo11_640_fire_11class": {"avg_ms": 59.08, "count": 177, "max_ms": 83.686, "p95_ms": 77.522}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.17, "count": 177, "max_ms": 98.939, "p95_ms": 92.87}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.151, "count": 178, "max_ms": 186.317, "p95_ms": 168.473}, "read_interval": {"avg_ms": 143.522, "count": 177, "max_ms": 194.906, "p95_ms": 177.755}, "triton_wait": {"avg_ms": 74.339, "count": 177, "max_ms": 117.249, "p95_ms": 100.107}, "visualize": {"avg_ms": 5.448, "count": 177, "max_ms": 8.161, "p95_ms": 6.604}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1715}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5988.609, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558e87b18100] error while decoding MB 36 27, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 102, "h264_decode_error_count": 331, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:15:08+09:00", "last_error_summary": "[h264 @ 0x558e87b18100] error while decoding MB 36 27, bytestream -5", "last_exit_at": "2026-07-06T20:12:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.604, "last_started_at": "2026-07-06T20:12:24+09:00", "pid": 25886, "process": {"exists": false, "pid": 25886}, "restart_count": 111, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 112}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.271, "sample_count": 2043, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 205871, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:15:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T20:15:08+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-06T20:15:08+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-06T20:15:08.138681+09:00", "mtime_epoch": 1783336508.1386807, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157541}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:15:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1636.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.906, "pushed": 6.906, "read": 6.906}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.628, "count": 206, "max_ms": 10.991, "p95_ms": 8.579}, "gstreamer_push": {"avg_ms": 1.071, "count": 207, "max_ms": 1.481, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.621, "count": 206, "max_ms": 82.645, "p95_ms": 71.947}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.957, "count": 206, "max_ms": 122.222, "p95_ms": 99.198}, "yolo11_640_fire_11class": {"avg_ms": 59.839, "count": 206, "max_ms": 90.328, "p95_ms": 78.983}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.074, "count": 206, "max_ms": 100.429, "p95_ms": 93.361}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.227, "count": 207, "max_ms": 180.875, "p95_ms": 170.282}, "read_interval": {"avg_ms": 145.294, "count": 207, "max_ms": 189.759, "p95_ms": 179.209}, "triton_wait": {"avg_ms": 74.168, "count": 206, "max_ms": 122.235, "p95_ms": 99.211}, "visualize": {"avg_ms": 5.583, "count": 206, "max_ms": 9.551, "p95_ms": 6.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1715}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5987.941, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:15:08+09:00", "last_error_summary": "[h264 @ 0x558e87b18100] error while decoding MB 36 27, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 102, "h264_decode_error_count": 331, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:15:08+09:00", "last_error_summary": "[h264 @ 0x558e87b18100] error while decoding MB 36 27, bytestream -5", "last_exit_at": "2026-07-06T20:15:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.684, "last_started_at": "2026-07-06T20:12:24+09:00", "pid": 25886, "process": {"exists": false, "pid": 25886}, "restart_count": 111, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 112}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.271, "sample_count": 2044, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 205872, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:15:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T20:15:08+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-06T20:15:08+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T20:15:08.138681+09:00", "mtime_epoch": 1783336508.1386807, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157541}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:15:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1636.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.922, "pushed": 6.922, "read": 6.889}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.625, "count": 207, "max_ms": 10.991, "p95_ms": 8.579}, "gstreamer_push": {"avg_ms": 1.071, "count": 207, "max_ms": 1.481, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.59, "count": 207, "max_ms": 82.645, "p95_ms": 71.947}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.928, "count": 207, "max_ms": 122.222, "p95_ms": 99.198}, "yolo11_640_fire_11class": {"avg_ms": 59.848, "count": 207, "max_ms": 90.328, "p95_ms": 78.983}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.053, "count": 207, "max_ms": 100.429, "p95_ms": 93.361}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.082, "count": 207, "max_ms": 180.875, "p95_ms": 170.282}, "read_interval": {"avg_ms": 145.202, "count": 206, "max_ms": 189.759, "p95_ms": 179.209}, "triton_wait": {"avg_ms": 74.138, "count": 207, "max_ms": 122.235, "p95_ms": 99.211}, "visualize": {"avg_ms": 5.58, "count": 207, "max_ms": 9.551, "p95_ms": 6.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1715}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5987.867, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 102, "h264_decode_error_count": 331, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:15:08+09:00", "last_error_summary": "[h264 @ 0x558e87b18100] error while decoding MB 36 27, bytestream -5", "last_exit_at": "2026-07-06T20:15:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.69, "last_started_at": "2026-07-06T20:12:24+09:00", "pid": 25886, "process": {"exists": false, "pid": 25886}, "restart_count": 111, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 112}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.271, "sample_count": 2044, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 205872, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:15:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T20:15:08+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T20:15:08+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-06T20:15:08.138681+09:00", "mtime_epoch": 1783336508.1386807, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157541}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:15:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1636.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.921, "pushed": 6.921, "read": 6.887}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.625, "count": 207, "max_ms": 10.991, "p95_ms": 8.579}, "gstreamer_push": {"avg_ms": 1.071, "count": 207, "max_ms": 1.481, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.59, "count": 207, "max_ms": 82.645, "p95_ms": 71.947}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.928, "count": 207, "max_ms": 122.222, "p95_ms": 99.198}, "yolo11_640_fire_11class": {"avg_ms": 59.848, "count": 207, "max_ms": 90.328, "p95_ms": 78.983}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.053, "count": 207, "max_ms": 100.429, "p95_ms": 93.361}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.082, "count": 207, "max_ms": 180.875, "p95_ms": 170.282}, "read_interval": {"avg_ms": 145.202, "count": 206, "max_ms": 189.759, "p95_ms": 179.209}, "triton_wait": {"avg_ms": 74.138, "count": 207, "max_ms": 122.235, "p95_ms": 99.211}, "visualize": {"avg_ms": 5.58, "count": 207, "max_ms": 9.551, "p95_ms": 6.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1715}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6093.699, "limit_mb": 8796093022207.996}, "detail": {"pid": 25993, "restart_count": 112, "start_count": 113}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 102, "h264_decode_error_count": 331, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:15:08+09:00", "last_error_summary": "[h264 @ 0x558e87b18100] error while decoding MB 36 27, bytestream -5", "last_exit_at": "2026-07-06T20:15:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:15:10+09:00", "pid": 25993, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.219, "name": "ffmpeg", "pid": 25993, "rss_mb": 139.219, "state": "S (sleeping)", "threads": 106}, "restart_count": 112, "restart_count_per_hour": 24, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 113}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.569, "sample_count": 2015, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 205872, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:15:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.342, "last_frame_pushed_at": "2026-07-06T20:15:08+09:00", "last_frame_read_age_sec": 4.467, "last_frame_read_at": "2026-07-06T20:15:08+09:00", "mjpeg": {"age_sec": 4.327, "exists": true, "mtime": "2026-07-06T20:15:08.262683+09:00", "mtime_epoch": 1783336508.2626827, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158180}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:15:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1636.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.902, "pushed": 5.902, "read": 5.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.649, "count": 176, "max_ms": 10.991, "p95_ms": 8.785}, "gstreamer_push": {"avg_ms": 1.067, "count": 177, "max_ms": 1.481, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.224, "count": 176, "max_ms": 82.645, "p95_ms": 71.788}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.459, "count": 176, "max_ms": 122.222, "p95_ms": 99.198}, "yolo11_640_fire_11class": {"avg_ms": 59.685, "count": 176, "max_ms": 90.328, "p95_ms": 77.734}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.607, "count": 176, "max_ms": 99.036, "p95_ms": 92.383}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.744, "count": 177, "max_ms": 180.875, "p95_ms": 172.83}, "read_interval": {"avg_ms": 145.822, "count": 176, "max_ms": 189.759, "p95_ms": 181.849}, "triton_wait": {"avg_ms": 73.691, "count": 176, "max_ms": 122.235, "p95_ms": 99.211}, "visualize": {"avg_ms": 5.649, "count": 176, "max_ms": 9.551, "p95_ms": 7.253}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1723}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.191, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56423ce97fc0] error while decoding MB 4 38, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 103, "h264_decode_error_count": 332, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:17:40+09:00", "last_error_summary": "[h264 @ 0x56423ce97fc0] error while decoding MB 4 38, bytestream -9", "last_exit_at": "2026-07-06T20:15:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.581, "last_started_at": "2026-07-06T20:15:10+09:00", "pid": 25993, "process": {"exists": false, "pid": 25993}, "restart_count": 112, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 113}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.56, "sample_count": 1944, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 206802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:17:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-06T20:17:40+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T20:17:40+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-06T20:17:40.129269+09:00", "mtime_epoch": 1783336660.1292691, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159329}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:17:40+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1636.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.346, "pushed": 6.346, "read": 6.346}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.178, "count": 189, "max_ms": 11.104, "p95_ms": 8.8}, "gstreamer_push": {"avg_ms": 1.023, "count": 190, "max_ms": 1.336, "p95_ms": 1.192}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.372, "count": 189, "max_ms": 84.602, "p95_ms": 74.766}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.057, "count": 189, "max_ms": 131.163, "p95_ms": 99.559}, "yolo11_640_fire_11class": {"avg_ms": 58.685, "count": 189, "max_ms": 85.437, "p95_ms": 77.148}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.444, "count": 189, "max_ms": 104.17, "p95_ms": 90.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.693, "count": 190, "max_ms": 211.863, "p95_ms": 181.952}, "read_interval": {"avg_ms": 158.043, "count": 190, "max_ms": 221.219, "p95_ms": 189.291}, "triton_wait": {"avg_ms": 73.135, "count": 189, "max_ms": 131.175, "p95_ms": 99.571}, "visualize": {"avg_ms": 6.323, "count": 189, "max_ms": 10.93, "p95_ms": 7.582}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1723}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5984.312, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:17:40+09:00", "last_error_summary": "[h264 @ 0x56423ce97fc0] error while decoding MB 4 38, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 103, "h264_decode_error_count": 332, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:17:40+09:00", "last_error_summary": "[h264 @ 0x56423ce97fc0] error while decoding MB 4 38, bytestream -9", "last_exit_at": "2026-07-06T20:17:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.684, "last_started_at": "2026-07-06T20:15:10+09:00", "pid": 25993, "process": {"exists": false, "pid": 25993}, "restart_count": 112, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 113}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.56, "sample_count": 1945, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 206803, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:17:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T20:17:40+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-06T20:17:40+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-06T20:17:40.129269+09:00", "mtime_epoch": 1783336660.1292691, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159329}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:17:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1636.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.356, "pushed": 6.356, "read": 6.323}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.173, "count": 190, "max_ms": 11.104, "p95_ms": 8.8}, "gstreamer_push": {"avg_ms": 1.024, "count": 190, "max_ms": 1.336, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.441, "count": 190, "max_ms": 84.602, "p95_ms": 74.766}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.17, "count": 190, "max_ms": 131.163, "p95_ms": 99.559}, "yolo11_640_fire_11class": {"avg_ms": 58.723, "count": 190, "max_ms": 85.437, "p95_ms": 77.148}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.485, "count": 190, "max_ms": 104.17, "p95_ms": 90.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.728, "count": 190, "max_ms": 211.863, "p95_ms": 181.952}, "read_interval": {"avg_ms": 158.095, "count": 189, "max_ms": 221.219, "p95_ms": 189.291}, "triton_wait": {"avg_ms": 73.248, "count": 190, "max_ms": 131.175, "p95_ms": 99.571}, "visualize": {"avg_ms": 6.317, "count": 190, "max_ms": 10.93, "p95_ms": 7.582}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1723}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.656, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 103, "h264_decode_error_count": 332, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:17:40+09:00", "last_error_summary": "[h264 @ 0x56423ce97fc0] error while decoding MB 4 38, bytestream -9", "last_exit_at": "2026-07-06T20:17:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.69, "last_started_at": "2026-07-06T20:15:10+09:00", "pid": 25993, "process": {"exists": false, "pid": 25993}, "restart_count": 112, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 113}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.56, "sample_count": 1945, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 206803, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:17:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T20:17:40+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-06T20:17:40+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-06T20:17:40.129269+09:00", "mtime_epoch": 1783336660.1292691, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159329}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:17:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1636.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.355, "pushed": 6.355, "read": 6.322}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.173, "count": 190, "max_ms": 11.104, "p95_ms": 8.8}, "gstreamer_push": {"avg_ms": 1.024, "count": 190, "max_ms": 1.336, "p95_ms": 1.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.441, "count": 190, "max_ms": 84.602, "p95_ms": 74.766}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.17, "count": 190, "max_ms": 131.163, "p95_ms": 99.559}, "yolo11_640_fire_11class": {"avg_ms": 58.723, "count": 190, "max_ms": 85.437, "p95_ms": 77.148}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.485, "count": 190, "max_ms": 104.17, "p95_ms": 90.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.728, "count": 190, "max_ms": 211.863, "p95_ms": 181.952}, "read_interval": {"avg_ms": 158.095, "count": 189, "max_ms": 221.219, "p95_ms": 189.291}, "triton_wait": {"avg_ms": 73.248, "count": 190, "max_ms": 131.175, "p95_ms": 99.571}, "visualize": {"avg_ms": 6.317, "count": 190, "max_ms": 10.93, "p95_ms": 7.582}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1723}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6101.602, "limit_mb": 8796093022207.996}, "detail": {"pid": 26100, "restart_count": 113, "start_count": 114}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 103, "h264_decode_error_count": 332, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:17:40+09:00", "last_error_summary": "[h264 @ 0x56423ce97fc0] error while decoding MB 4 38, bytestream -9", "last_exit_at": "2026-07-06T20:17:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:17:43+09:00", "pid": 26100, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.789, "name": "ffmpeg", "pid": 26100, "rss_mb": 141.789, "state": "S (sleeping)", "threads": 106}, "restart_count": 113, "restart_count_per_hour": 24, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 114}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 177.477, "sample_count": 1910, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 206803, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:17:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.91, "last_frame_pushed_at": "2026-07-06T20:17:40+09:00", "last_frame_read_age_sec": 5.059, "last_frame_read_at": "2026-07-06T20:17:40+09:00", "mjpeg": {"age_sec": 4.893, "exists": true, "mtime": "2026-07-06T20:17:40.285272+09:00", "mtime_epoch": 1783336660.285272, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159038}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:17:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1636.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 157, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.284, "pushed": 5.284, "read": 5.25}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.094, "count": 158, "max_ms": 10.958, "p95_ms": 8.35}, "gstreamer_push": {"avg_ms": 1.023, "count": 158, "max_ms": 1.336, "p95_ms": 1.192}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.515, "count": 158, "max_ms": 84.602, "p95_ms": 75.112}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.442, "count": 158, "max_ms": 131.163, "p95_ms": 100.075}, "yolo11_640_fire_11class": {"avg_ms": 58.846, "count": 158, "max_ms": 85.437, "p95_ms": 80.251}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.606, "count": 158, "max_ms": 104.17, "p95_ms": 90.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.789, "count": 158, "max_ms": 211.863, "p95_ms": 183.208}, "read_interval": {"avg_ms": 159.17, "count": 157, "max_ms": 221.219, "p95_ms": 190.918}, "triton_wait": {"avg_ms": 73.522, "count": 158, "max_ms": 131.175, "p95_ms": 100.088}, "visualize": {"avg_ms": 6.312, "count": 158, "max_ms": 10.93, "p95_ms": 7.275}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1731}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5994.695, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55736713c280] error while decoding MB 63 28, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 104, "h264_decode_error_count": 333, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:20:17+09:00", "last_error_summary": "[h264 @ 0x55736713c280] error while decoding MB 63 28, bytestream -7", "last_exit_at": "2026-07-06T20:17:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 154.46, "last_started_at": "2026-07-06T20:17:43+09:00", "pid": 26100, "process": {"exists": false, "pid": 26100}, "restart_count": 113, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 114}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.199, "sample_count": 1874, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 207777, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:20:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-06T20:20:17+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T20:20:17+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-06T20:20:17.599981+09:00", "mtime_epoch": 1783336817.5999808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165648}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:20:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1637.031, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.343, "pushed": 6.343, "read": 6.343}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.058, "count": 190, "max_ms": 10.814, "p95_ms": 9.115}, "gstreamer_push": {"avg_ms": 1.047, "count": 190, "max_ms": 1.317, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.871, "count": 190, "max_ms": 86.119, "p95_ms": 77.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.344, "count": 190, "max_ms": 126.208, "p95_ms": 103.011}, "yolo11_640_fire_11class": {"avg_ms": 59.631, "count": 190, "max_ms": 87.377, "p95_ms": 82.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.613, "count": 190, "max_ms": 105.773, "p95_ms": 97.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.061, "count": 190, "max_ms": 212.104, "p95_ms": 184.031}, "read_interval": {"avg_ms": 158.077, "count": 190, "max_ms": 222.712, "p95_ms": 195.544}, "triton_wait": {"avg_ms": 74.757, "count": 190, "max_ms": 126.222, "p95_ms": 103.022}, "visualize": {"avg_ms": 6.158, "count": 190, "max_ms": 8.539, "p95_ms": 7.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1731}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.949, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:20:17+09:00", "last_error_summary": "[h264 @ 0x55736713c280] error while decoding MB 63 28, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 104, "h264_decode_error_count": 333, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:20:17+09:00", "last_error_summary": "[h264 @ 0x55736713c280] error while decoding MB 63 28, bytestream -7", "last_exit_at": "2026-07-06T20:20:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 154.56, "last_started_at": "2026-07-06T20:17:43+09:00", "pid": 26100, "process": {"exists": false, "pid": 26100}, "restart_count": 113, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 114}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.199, "sample_count": 1874, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 207778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:20:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T20:20:17+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-06T20:20:17+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-06T20:20:17.599981+09:00", "mtime_epoch": 1783336817.5999808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165648}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:20:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1637.031, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.355, "pushed": 6.355, "read": 6.322}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.057, "count": 190, "max_ms": 10.814, "p95_ms": 9.115}, "gstreamer_push": {"avg_ms": 1.048, "count": 190, "max_ms": 1.317, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.941, "count": 190, "max_ms": 86.119, "p95_ms": 77.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.443, "count": 190, "max_ms": 126.208, "p95_ms": 103.011}, "yolo11_640_fire_11class": {"avg_ms": 59.636, "count": 190, "max_ms": 87.377, "p95_ms": 82.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.655, "count": 190, "max_ms": 105.773, "p95_ms": 97.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.141, "count": 190, "max_ms": 212.104, "p95_ms": 184.031}, "read_interval": {"avg_ms": 158.17, "count": 189, "max_ms": 222.712, "p95_ms": 195.544}, "triton_wait": {"avg_ms": 74.856, "count": 190, "max_ms": 126.222, "p95_ms": 103.022}, "visualize": {"avg_ms": 6.158, "count": 190, "max_ms": 8.539, "p95_ms": 7.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1731}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.977, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 104, "h264_decode_error_count": 333, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:20:17+09:00", "last_error_summary": "[h264 @ 0x55736713c280] error while decoding MB 63 28, bytestream -7", "last_exit_at": "2026-07-06T20:20:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 154.565, "last_started_at": "2026-07-06T20:17:43+09:00", "pid": 26100, "process": {"exists": false, "pid": 26100}, "restart_count": 113, "restart_count_per_hour": 23, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 114}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.199, "sample_count": 1874, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 207778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:20:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T20:20:17+09:00", "last_frame_read_age_sec": 0.157, "last_frame_read_at": "2026-07-06T20:20:17+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-06T20:20:17.599981+09:00", "mtime_epoch": 1783336817.5999808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165648}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:20:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1637.031, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.354, "pushed": 6.354, "read": 6.32}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.057, "count": 190, "max_ms": 10.814, "p95_ms": 9.115}, "gstreamer_push": {"avg_ms": 1.048, "count": 190, "max_ms": 1.317, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.941, "count": 190, "max_ms": 86.119, "p95_ms": 77.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.443, "count": 190, "max_ms": 126.208, "p95_ms": 103.011}, "yolo11_640_fire_11class": {"avg_ms": 59.636, "count": 190, "max_ms": 87.377, "p95_ms": 82.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.655, "count": 190, "max_ms": 105.773, "p95_ms": 97.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.141, "count": 190, "max_ms": 212.104, "p95_ms": 184.031}, "read_interval": {"avg_ms": 158.17, "count": 189, "max_ms": 222.712, "p95_ms": 195.544}, "triton_wait": {"avg_ms": 74.856, "count": 190, "max_ms": 126.222, "p95_ms": 103.022}, "visualize": {"avg_ms": 6.158, "count": 190, "max_ms": 8.539, "p95_ms": 7.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1731}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6099.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 26207, "restart_count": 114, "start_count": 115}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 104, "h264_decode_error_count": 333, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:20:17+09:00", "last_error_summary": "[h264 @ 0x55736713c280] error while decoding MB 63 28, bytestream -7", "last_exit_at": "2026-07-06T20:20:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:20:20+09:00", "pid": 26207, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.215, "name": "ffmpeg", "pid": 26207, "rss_mb": 136.215, "state": "S (sleeping)", "threads": 106}, "restart_count": 114, "restart_count_per_hour": 24, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 115}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.298, "sample_count": 1842, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 207778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:20:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.718, "last_frame_pushed_at": "2026-07-06T20:20:17+09:00", "last_frame_read_age_sec": 4.865, "last_frame_read_at": "2026-07-06T20:20:17+09:00", "mjpeg": {"age_sec": 4.695, "exists": true, "mtime": "2026-07-06T20:20:17.755984+09:00", "mtime_epoch": 1783336817.7559836, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165457}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:20:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1637.031, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 159, "pushed_frames": 159, "read_frames": 158, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.312, "pushed": 5.312, "read": 5.279}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.151, "count": 159, "max_ms": 10.814, "p95_ms": 9.682}, "gstreamer_push": {"avg_ms": 1.046, "count": 159, "max_ms": 1.317, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.451, "count": 159, "max_ms": 86.119, "p95_ms": 77.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.49, "count": 159, "max_ms": 126.208, "p95_ms": 103.011}, "yolo11_640_fire_11class": {"avg_ms": 59.45, "count": 159, "max_ms": 87.377, "p95_ms": 82.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.519, "count": 159, "max_ms": 101.733, "p95_ms": 97.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.114, "count": 159, "max_ms": 212.104, "p95_ms": 186.415}, "read_interval": {"avg_ms": 159.822, "count": 158, "max_ms": 222.712, "p95_ms": 196.11}, "triton_wait": {"avg_ms": 74.772, "count": 159, "max_ms": 126.222, "p95_ms": 103.022}, "visualize": {"avg_ms": 6.275, "count": 159, "max_ms": 8.539, "p95_ms": 7.647}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1756}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5996.914, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f6f645d3c0] error while decoding MB 69 24, bytestream -5 [h264 @ 0x55f6f633b480] left block unavailable for requested intra mode [h264 @ 0x55f6f633b480] error while decoding MB 0 31, bytestream 56618 [rtsp @ 0x55f6f6320f00] RTP: PT=60: bad cseq 5254 expected=aeb5 [h264 @ 0x55f6f6485740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f6f6485740] error while decoding MB 0 16, bytestream 5243 [h264 @ 0x55f6f6cd7080] error while decoding MB 16 12, bytestream -10 [in#0/", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 105, "h264_decode_error_count": 337, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:26:33+09:00", "last_error_summary": "[h264 @ 0x55f6f645d3c0] error while decoding MB 69 24, bytestream -5 [h264 @ 0x55f6f633b480] left block unavailable for requested intra mode [h264 @ 0x55f6f633b480] error while decoding MB 0 31, bytestream 56618 [rtsp @ 0x55f6f6320f00] RTP: PT=60: bad cseq 5254 expected=aeb5 [h264 @ 0x55f6f6485740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f6f6485740] error while decoding MB 0 16, bytestream 5243 [h264 @ 0x55f6f6cd7080] error while decoding MB 16 12, bytestream -10 [in#0/", "last_exit_at": "2026-07-06T20:20:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 373.343, "last_started_at": "2026-07-06T20:20:20+09:00", "pid": 26207, "process": {"exists": false, "pid": 26207}, "restart_count": 114, "restart_count_per_hour": 21, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 115}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.46, "sample_count": 2410, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 210740, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:26:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T20:26:33+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T20:26:33+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T20:26:33.750690+09:00", "mtime_epoch": 1783337193.7506902, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151401}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:26:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1638.578, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 92, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.072, "pushed": 3.072, "read": 3.072}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.979, "count": 92, "max_ms": 7.21, "p95_ms": 6.332}, "gstreamer_push": {"avg_ms": 1.3, "count": 92, "max_ms": 1.908, "p95_ms": 1.467}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.389, "count": 92, "max_ms": 82.262, "p95_ms": 79.041}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.869, "count": 92, "max_ms": 98.993, "p95_ms": 94.69}, "yolo11_640_fire_11class": {"avg_ms": 71.009, "count": 92, "max_ms": 83.859, "p95_ms": 78.427}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.269, "count": 92, "max_ms": 98.216, "p95_ms": 95.211}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.776, "count": 92, "max_ms": 150.313, "p95_ms": 139.325}, "read_interval": {"avg_ms": 326.081, "count": 92, "max_ms": 10132.101, "p95_ms": 155.006}, "triton_wait": {"avg_ms": 85.447, "count": 92, "max_ms": 99.007, "p95_ms": 95.391}, "visualize": {"avg_ms": 3.869, "count": 92, "max_ms": 6.511, "p95_ms": 5.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1756}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5995.922, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:26:33+09:00", "last_error_summary": "[h264 @ 0x55f6f645d3c0] error while decoding MB 69 24, bytestream -5 [h264 @ 0x55f6f633b480] left block unavailable for requested intra mode [h264 @ 0x55f6f633b480] error while decoding MB 0 31, bytestream 56618 [rtsp @ 0x55f6f6320f00] RTP: PT=60: bad cseq 5254 expected=aeb5 [h264 @ 0x55f6f6485740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f6f6485740] error while decoding MB 0 16, bytestream 5243 [h264 @ 0x55f6f6cd7080] error while decoding MB 16 12, bytestream -10 [in#0/", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 105, "h264_decode_error_count": 337, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:26:33+09:00", "last_error_summary": "[h264 @ 0x55f6f645d3c0] error while decoding MB 69 24, bytestream -5 [h264 @ 0x55f6f633b480] left block unavailable for requested intra mode [h264 @ 0x55f6f633b480] error while decoding MB 0 31, bytestream 56618 [rtsp @ 0x55f6f6320f00] RTP: PT=60: bad cseq 5254 expected=aeb5 [h264 @ 0x55f6f6485740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f6f6485740] error while decoding MB 0 16, bytestream 5243 [h264 @ 0x55f6f6cd7080] error while decoding MB 16 12, bytestream -10 [in#0/", "last_exit_at": "2026-07-06T20:26:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 373.409, "last_started_at": "2026-07-06T20:20:20+09:00", "pid": 26207, "process": {"exists": false, "pid": 26207}, "restart_count": 114, "restart_count_per_hour": 21, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 115}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.46, "sample_count": 2410, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 210741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:26:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T20:26:33+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-06T20:26:33+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T20:26:33.750690+09:00", "mtime_epoch": 1783337193.7506902, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151401}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:26:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1638.578, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 92, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.067, "pushed": 3.067, "read": 3.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.975, "count": 92, "max_ms": 7.21, "p95_ms": 6.332}, "gstreamer_push": {"avg_ms": 1.3, "count": 92, "max_ms": 1.908, "p95_ms": 1.467}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.148, "count": 92, "max_ms": 82.262, "p95_ms": 78.854}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.899, "count": 92, "max_ms": 98.993, "p95_ms": 94.69}, "yolo11_640_fire_11class": {"avg_ms": 71.1, "count": 92, "max_ms": 85.201, "p95_ms": 78.623}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.305, "count": 92, "max_ms": 98.216, "p95_ms": 95.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.803, "count": 92, "max_ms": 150.313, "p95_ms": 139.325}, "read_interval": {"avg_ms": 326.081, "count": 92, "max_ms": 10132.101, "p95_ms": 155.006}, "triton_wait": {"avg_ms": 85.478, "count": 92, "max_ms": 99.007, "p95_ms": 95.716}, "visualize": {"avg_ms": 3.869, "count": 92, "max_ms": 6.511, "p95_ms": 5.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1756}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5995.805, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 105, "h264_decode_error_count": 337, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:26:33+09:00", "last_error_summary": "[h264 @ 0x55f6f645d3c0] error while decoding MB 69 24, bytestream -5 [h264 @ 0x55f6f633b480] left block unavailable for requested intra mode [h264 @ 0x55f6f633b480] error while decoding MB 0 31, bytestream 56618 [rtsp @ 0x55f6f6320f00] RTP: PT=60: bad cseq 5254 expected=aeb5 [h264 @ 0x55f6f6485740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f6f6485740] error while decoding MB 0 16, bytestream 5243 [h264 @ 0x55f6f6cd7080] error while decoding MB 16 12, bytestream -10 [in#0/", "last_exit_at": "2026-07-06T20:26:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 373.42, "last_started_at": "2026-07-06T20:20:20+09:00", "pid": 26207, "process": {"exists": false, "pid": 26207}, "restart_count": 114, "restart_count_per_hour": 21, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 115}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.46, "sample_count": 2410, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 210741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:26:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-06T20:26:33+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T20:26:33+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T20:26:33.750690+09:00", "mtime_epoch": 1783337193.7506902, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151401}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:26:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1638.578, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.075, "pushed": 3.075, "read": 3.042}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.975, "count": 92, "max_ms": 7.21, "p95_ms": 6.332}, "gstreamer_push": {"avg_ms": 1.3, "count": 92, "max_ms": 1.908, "p95_ms": 1.467}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.148, "count": 92, "max_ms": 82.262, "p95_ms": 78.854}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.899, "count": 92, "max_ms": 98.993, "p95_ms": 94.69}, "yolo11_640_fire_11class": {"avg_ms": 71.1, "count": 92, "max_ms": 85.201, "p95_ms": 78.623}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.305, "count": 92, "max_ms": 98.216, "p95_ms": 95.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.803, "count": 92, "max_ms": 150.313, "p95_ms": 139.325}, "read_interval": {"avg_ms": 328.342, "count": 91, "max_ms": 10132.101, "p95_ms": 155.006}, "triton_wait": {"avg_ms": 85.478, "count": 92, "max_ms": 99.007, "p95_ms": 95.716}, "visualize": {"avg_ms": 3.869, "count": 92, "max_ms": 6.511, "p95_ms": 5.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1756}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6094.59, "limit_mb": 8796093022207.996}, "detail": {"pid": 26314, "restart_count": 115, "start_count": 116}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 105, "h264_decode_error_count": 337, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:26:33+09:00", "last_error_summary": "[h264 @ 0x55f6f645d3c0] error while decoding MB 69 24, bytestream -5 [h264 @ 0x55f6f633b480] left block unavailable for requested intra mode [h264 @ 0x55f6f633b480] error while decoding MB 0 31, bytestream 56618 [rtsp @ 0x55f6f6320f00] RTP: PT=60: bad cseq 5254 expected=aeb5 [h264 @ 0x55f6f6485740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f6f6485740] error while decoding MB 0 16, bytestream 5243 [h264 @ 0x55f6f6cd7080] error while decoding MB 16 12, bytestream -10 [in#0/", "last_exit_at": "2026-07-06T20:26:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:26:36+09:00", "pid": 26314, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.512, "name": "ffmpeg", "pid": 26314, "rss_mb": 138.512, "state": "S (sleeping)", "threads": 106}, "restart_count": 115, "restart_count_per_hour": 22, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 116}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.478, "sample_count": 2372, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 210741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:26:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.445, "last_frame_pushed_at": "2026-07-06T20:26:33+09:00", "last_frame_read_age_sec": 4.558, "last_frame_read_at": "2026-07-06T20:26:33+09:00", "mjpeg": {"age_sec": 4.43, "exists": true, "mtime": "2026-07-06T20:26:33.866692+09:00", "mtime_epoch": 1783337193.8666923, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 145882}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:26:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1638.578, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.839, "pushed": 1.839, "read": 1.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.827, "count": 55, "max_ms": 6.164, "p95_ms": 6.064}, "gstreamer_push": {"avg_ms": 1.322, "count": 55, "max_ms": 1.908, "p95_ms": 1.498}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.67, "count": 55, "max_ms": 79.448, "p95_ms": 78.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.834, "count": 55, "max_ms": 98.993, "p95_ms": 94.603}, "yolo11_640_fire_11class": {"avg_ms": 71.481, "count": 55, "max_ms": 85.201, "p95_ms": 78.623}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.145, "count": 55, "max_ms": 98.216, "p95_ms": 95.91}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.319, "count": 55, "max_ms": 141.219, "p95_ms": 111.446}, "read_interval": {"avg_ms": 471.371, "count": 54, "max_ms": 10132.101, "p95_ms": 149.369}, "triton_wait": {"avg_ms": 85.772, "count": 55, "max_ms": 99.007, "p95_ms": 96.308}, "visualize": {"avg_ms": 3.746, "count": 55, "max_ms": 5.112, "p95_ms": 4.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1773}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6002.453, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f4ca7137c0] reference picture missing during reorder [h264 @ 0x55f4ca7137c0] Missing reference picture, default is 65559 [h264 @ 0x55f4ca7e2640] mmco: unref short failure [h264 @ 0x55f4ca7e2640] illegal short term buffer state detected [h264 @ 0x55f4c9b71580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9b71580] error while decoding MB 0 31, bytestream 4472 [h264 @ 0x55f4c9aaaac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9aaaac0] er", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 341, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:31:05+09:00", "last_error_summary": "[h264 @ 0x55f4ca7137c0] reference picture missing during reorder [h264 @ 0x55f4ca7137c0] Missing reference picture, default is 65559 [h264 @ 0x55f4ca7e2640] mmco: unref short failure [h264 @ 0x55f4ca7e2640] illegal short term buffer state detected [h264 @ 0x55f4c9b71580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9b71580] error while decoding MB 0 31, bytestream 4472 [h264 @ 0x55f4c9aaaac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9aaaac0] er", "last_exit_at": "2026-07-06T20:26:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 268.922, "last_started_at": "2026-07-06T20:26:36+09:00", "pid": 26314, "process": {"exists": false, "pid": 26314}, "restart_count": 115, "restart_count_per_hour": 20, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 116}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.959, "sample_count": 2105, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 212764, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:31:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T20:31:05+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-06T20:31:05+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T20:31:05.175606+09:00", "mtime_epoch": 1783337465.1756065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168012}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:31:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1639.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 75, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.5, "pushed": 2.5, "read": 2.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.901, "count": 74, "max_ms": 10.523, "p95_ms": 8.583}, "gstreamer_push": {"avg_ms": 1.15, "count": 75, "max_ms": 2.265, "p95_ms": 1.387}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.34, "count": 74, "max_ms": 80.61, "p95_ms": 75.018}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.665, "count": 74, "max_ms": 113.976, "p95_ms": 98.933}, "yolo11_640_fire_11class": {"avg_ms": 62.262, "count": 74, "max_ms": 86.518, "p95_ms": 82.328}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.766, "count": 74, "max_ms": 103.966, "p95_ms": 99.59}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.442, "count": 75, "max_ms": 168.543, "p95_ms": 153.084}, "read_interval": {"avg_ms": 400.92, "count": 75, "max_ms": 10130.411, "p95_ms": 171.25}, "triton_wait": {"avg_ms": 77.569, "count": 74, "max_ms": 113.99, "p95_ms": 102.06}, "visualize": {"avg_ms": 4.999, "count": 74, "max_ms": 8.491, "p95_ms": 6.83}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1773}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6001.574, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:31:05+09:00", "last_error_summary": "[h264 @ 0x55f4ca7137c0] reference picture missing during reorder [h264 @ 0x55f4ca7137c0] Missing reference picture, default is 65559 [h264 @ 0x55f4ca7e2640] mmco: unref short failure [h264 @ 0x55f4ca7e2640] illegal short term buffer state detected [h264 @ 0x55f4c9b71580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9b71580] error while decoding MB 0 31, bytestream 4472 [h264 @ 0x55f4c9aaaac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9aaaac0] er", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 341, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:31:05+09:00", "last_error_summary": "[h264 @ 0x55f4ca7137c0] reference picture missing during reorder [h264 @ 0x55f4ca7137c0] Missing reference picture, default is 65559 [h264 @ 0x55f4ca7e2640] mmco: unref short failure [h264 @ 0x55f4ca7e2640] illegal short term buffer state detected [h264 @ 0x55f4c9b71580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9b71580] error while decoding MB 0 31, bytestream 4472 [h264 @ 0x55f4c9aaaac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9aaaac0] er", "last_exit_at": "2026-07-06T20:31:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 268.988, "last_started_at": "2026-07-06T20:26:36+09:00", "pid": 26314, "process": {"exists": false, "pid": 26314}, "restart_count": 115, "restart_count_per_hour": 20, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 116}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.959, "sample_count": 2106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 212765, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:31:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T20:31:05+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T20:31:05+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T20:31:05.175606+09:00", "mtime_epoch": 1783337465.1756065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168012}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:31:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1639.609, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 74, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.505, "pushed": 2.505, "read": 2.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.874, "count": 75, "max_ms": 10.523, "p95_ms": 8.583}, "gstreamer_push": {"avg_ms": 1.153, "count": 75, "max_ms": 2.265, "p95_ms": 1.405}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.447, "count": 75, "max_ms": 80.61, "p95_ms": 75.018}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.924, "count": 75, "max_ms": 113.976, "p95_ms": 98.933}, "yolo11_640_fire_11class": {"avg_ms": 62.467, "count": 75, "max_ms": 86.518, "p95_ms": 82.328}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.057, "count": 75, "max_ms": 103.966, "p95_ms": 99.59}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.224, "count": 75, "max_ms": 168.543, "p95_ms": 153.084}, "read_interval": {"avg_ms": 404.52, "count": 74, "max_ms": 10130.411, "p95_ms": 171.25}, "triton_wait": {"avg_ms": 77.817, "count": 75, "max_ms": 113.99, "p95_ms": 102.06}, "visualize": {"avg_ms": 4.986, "count": 75, "max_ms": 8.491, "p95_ms": 6.83}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1773}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6001.742, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 341, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:31:05+09:00", "last_error_summary": "[h264 @ 0x55f4ca7137c0] reference picture missing during reorder [h264 @ 0x55f4ca7137c0] Missing reference picture, default is 65559 [h264 @ 0x55f4ca7e2640] mmco: unref short failure [h264 @ 0x55f4ca7e2640] illegal short term buffer state detected [h264 @ 0x55f4c9b71580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9b71580] error while decoding MB 0 31, bytestream 4472 [h264 @ 0x55f4c9aaaac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9aaaac0] er", "last_exit_at": "2026-07-06T20:31:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 268.996, "last_started_at": "2026-07-06T20:26:36+09:00", "pid": 26314, "process": {"exists": false, "pid": 26314}, "restart_count": 115, "restart_count_per_hour": 20, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 116}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.959, "sample_count": 2106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 212765, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:31:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T20:31:05+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T20:31:05+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T20:31:05.175606+09:00", "mtime_epoch": 1783337465.1756065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168012}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:31:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1639.609, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 74, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.504, "pushed": 2.504, "read": 2.471}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.874, "count": 75, "max_ms": 10.523, "p95_ms": 8.583}, "gstreamer_push": {"avg_ms": 1.153, "count": 75, "max_ms": 2.265, "p95_ms": 1.405}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.447, "count": 75, "max_ms": 80.61, "p95_ms": 75.018}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.924, "count": 75, "max_ms": 113.976, "p95_ms": 98.933}, "yolo11_640_fire_11class": {"avg_ms": 62.467, "count": 75, "max_ms": 86.518, "p95_ms": 82.328}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.057, "count": 75, "max_ms": 103.966, "p95_ms": 99.59}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.224, "count": 75, "max_ms": 168.543, "p95_ms": 153.084}, "read_interval": {"avg_ms": 404.52, "count": 74, "max_ms": 10130.411, "p95_ms": 171.25}, "triton_wait": {"avg_ms": 77.817, "count": 75, "max_ms": 113.99, "p95_ms": 102.06}, "visualize": {"avg_ms": 4.986, "count": 75, "max_ms": 8.491, "p95_ms": 6.83}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1773}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6115.02, "limit_mb": 8796093022207.996}, "detail": {"pid": 26421, "restart_count": 116, "start_count": 117}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 341, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:31:05+09:00", "last_error_summary": "[h264 @ 0x55f4ca7137c0] reference picture missing during reorder [h264 @ 0x55f4ca7137c0] Missing reference picture, default is 65559 [h264 @ 0x55f4ca7e2640] mmco: unref short failure [h264 @ 0x55f4ca7e2640] illegal short term buffer state detected [h264 @ 0x55f4c9b71580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9b71580] error while decoding MB 0 31, bytestream 4472 [h264 @ 0x55f4c9aaaac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4c9aaaac0] er", "last_exit_at": "2026-07-06T20:31:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:31:07+09:00", "pid": 26421, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.531, "name": "ffmpeg", "pid": 26421, "rss_mb": 139.531, "state": "S (sleeping)", "threads": 106}, "restart_count": 116, "restart_count_per_hour": 21, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 117}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.228, "sample_count": 2065, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 212765, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:31:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.609, "last_frame_pushed_at": "2026-07-06T20:31:05+09:00", "last_frame_read_age_sec": 4.721, "last_frame_read_at": "2026-07-06T20:31:05+09:00", "mjpeg": {"age_sec": 4.591, "exists": true, "mtime": "2026-07-06T20:31:05.295609+09:00", "mtime_epoch": 1783337465.2956088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168139}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:31:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1639.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 40, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.373, "pushed": 1.373, "read": 1.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.524, "count": 41, "max_ms": 6.95, "p95_ms": 6.841}, "gstreamer_push": {"avg_ms": 1.146, "count": 41, "max_ms": 1.447, "p95_ms": 1.387}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.444, "count": 41, "max_ms": 80.61, "p95_ms": 74.954}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.148, "count": 41, "max_ms": 113.976, "p95_ms": 99.79}, "yolo11_640_fire_11class": {"avg_ms": 63.73, "count": 41, "max_ms": 86.518, "p95_ms": 85.222}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.033, "count": 41, "max_ms": 103.966, "p95_ms": 100.385}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.636, "count": 41, "max_ms": 162.588, "p95_ms": 152.359}, "read_interval": {"avg_ms": 632.08, "count": 40, "max_ms": 10130.411, "p95_ms": 171.25}, "triton_wait": {"avg_ms": 79.32, "count": 41, "max_ms": 113.99, "p95_ms": 103.979}, "visualize": {"avg_ms": 4.755, "count": 41, "max_ms": 5.723, "p95_ms": 5.552}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2022}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5938.023, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55627f27cfc0] error while decoding MB 102 4, bytestream -15 [in#0/rtsp @ 0x55627f1fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55627f1fbe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:32:24+09:00", "last_error_summary": "[h264 @ 0x55627f27cfc0] error while decoding MB 102 4, bytestream -15 [in#0/rtsp @ 0x55627f1fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55627f1fbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T17:01:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 12645.793, "last_started_at": "2026-07-06T17:01:38+09:00", "pid": 18866, "process": {"exists": false, "pid": 18866}, "restart_count": 46, "restart_count_per_hour": 0, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.235, "sample_count": 2752, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 242735, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:32:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-06T20:32:24+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T20:32:24+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T20:32:24.605039+09:00", "mtime_epoch": 1783337544.605039, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97615}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:32:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1527.52, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 50, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.667, "pushed": 1.667, "read": 1.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.472, "count": 49, "max_ms": 7.124, "p95_ms": 6.932}, "gstreamer_push": {"avg_ms": 1.351, "count": 50, "max_ms": 1.916, "p95_ms": 1.769}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.355, "count": 49, "max_ms": 82.567, "p95_ms": 78.386}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 88.119, "count": 49, "max_ms": 105.389, "p95_ms": 103.231}, "yolo11_640_fire_11class": {"avg_ms": 71.838, "count": 49, "max_ms": 88.185, "p95_ms": 83.772}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.715, "count": 49, "max_ms": 98.907, "p95_ms": 96.413}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.219, "count": 50, "max_ms": 124.955, "p95_ms": 118.057}, "read_interval": {"avg_ms": 600.967, "count": 50, "max_ms": 24357.881, "p95_ms": 132.373}, "triton_wait": {"avg_ms": 88.328, "count": 49, "max_ms": 105.399, "p95_ms": 103.243}, "visualize": {"avg_ms": 3.966, "count": 49, "max_ms": 6.159, "p95_ms": 6.107}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2022}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5936.844, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:32:24+09:00", "last_error_summary": "[h264 @ 0x55627f27cfc0] error while decoding MB 102 4, bytestream -15 [in#0/rtsp @ 0x55627f1fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55627f1fbe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:32:24+09:00", "last_error_summary": "[h264 @ 0x55627f27cfc0] error while decoding MB 102 4, bytestream -15 [in#0/rtsp @ 0x55627f1fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55627f1fbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:32:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 12645.849, "last_started_at": "2026-07-06T17:01:38+09:00", "pid": 18866, "process": {"exists": false, "pid": 18866}, "restart_count": 46, "restart_count_per_hour": 0, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.235, "sample_count": 2753, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 242736, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:32:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T20:32:24+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-06T20:32:24+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T20:32:24.605039+09:00", "mtime_epoch": 1783337544.605039, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97615}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:32:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1527.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.67, "pushed": 1.67, "read": 1.637}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.463, "count": 50, "max_ms": 7.124, "p95_ms": 6.932}, "gstreamer_push": {"avg_ms": 1.348, "count": 50, "max_ms": 1.916, "p95_ms": 1.769}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.658, "count": 50, "max_ms": 82.567, "p95_ms": 80.102}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 88.164, "count": 50, "max_ms": 105.389, "p95_ms": 103.231}, "yolo11_640_fire_11class": {"avg_ms": 72.039, "count": 50, "max_ms": 88.185, "p95_ms": 83.772}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.982, "count": 50, "max_ms": 98.907, "p95_ms": 97.055}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.185, "count": 50, "max_ms": 124.955, "p95_ms": 118.057}, "read_interval": {"avg_ms": 610.637, "count": 49, "max_ms": 24357.881, "p95_ms": 132.373}, "triton_wait": {"avg_ms": 88.503, "count": 50, "max_ms": 105.399, "p95_ms": 103.243}, "visualize": {"avg_ms": 3.957, "count": 50, "max_ms": 6.159, "p95_ms": 6.107}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2022}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5936.961, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:32:24+09:00", "last_error_summary": "[h264 @ 0x55627f27cfc0] error while decoding MB 102 4, bytestream -15 [in#0/rtsp @ 0x55627f1fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55627f1fbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:32:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 12645.857, "last_started_at": "2026-07-06T17:01:38+09:00", "pid": 18866, "process": {"exists": false, "pid": 18866}, "restart_count": 46, "restart_count_per_hour": 0, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.235, "sample_count": 2753, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 242736, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:32:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T20:32:24+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T20:32:24+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T20:32:24.605039+09:00", "mtime_epoch": 1783337544.605039, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97615}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:32:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1527.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.67, "pushed": 1.67, "read": 1.636}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.463, "count": 50, "max_ms": 7.124, "p95_ms": 6.932}, "gstreamer_push": {"avg_ms": 1.348, "count": 50, "max_ms": 1.916, "p95_ms": 1.769}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.658, "count": 50, "max_ms": 82.567, "p95_ms": 80.102}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 88.164, "count": 50, "max_ms": 105.389, "p95_ms": 103.231}, "yolo11_640_fire_11class": {"avg_ms": 72.039, "count": 50, "max_ms": 88.185, "p95_ms": 83.772}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.982, "count": 50, "max_ms": 98.907, "p95_ms": 97.055}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.185, "count": 50, "max_ms": 124.955, "p95_ms": 118.057}, "read_interval": {"avg_ms": 610.637, "count": 49, "max_ms": 24357.881, "p95_ms": 132.373}, "triton_wait": {"avg_ms": 88.503, "count": 50, "max_ms": 105.399, "p95_ms": 103.243}, "visualize": {"avg_ms": 3.957, "count": 50, "max_ms": 6.159, "p95_ms": 6.107}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2022}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6107.645, "limit_mb": 8796093022207.996}, "detail": {"pid": 26528, "restart_count": 47, "start_count": 48}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:32:24+09:00", "last_error_summary": "[h264 @ 0x55627f27cfc0] error while decoding MB 102 4, bytestream -15 [in#0/rtsp @ 0x55627f1fbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55627f1fbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:32:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:32:27+09:00", "pid": 26528, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.207, "name": "ffmpeg", "pid": 26528, "rss_mb": 209.207, "state": "S (sleeping)", "threads": 106}, "restart_count": 47, "restart_count_per_hour": 1, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.221, "sample_count": 2710, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 242736, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:32:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.754, "last_frame_pushed_at": "2026-07-06T20:32:24+09:00", "last_frame_read_age_sec": 4.865, "last_frame_read_at": "2026-07-06T20:32:24+09:00", "mjpeg": {"age_sec": 4.739, "exists": true, "mtime": "2026-07-06T20:32:24.725041+09:00", "mtime_epoch": 1783337544.7250412, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95537}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:32:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1527.52, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.105, "pushed": 2.105, "read": 2.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.119, "count": 13, "max_ms": 4.287, "p95_ms": 4.287}, "gstreamer_push": {"avg_ms": 1.312, "count": 13, "max_ms": 1.399, "p95_ms": 1.399}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.719, "count": 13, "max_ms": 82.474, "p95_ms": 82.474}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.463, "count": 13, "max_ms": 96.647, "p95_ms": 96.647}, "yolo11_640_fire_11class": {"avg_ms": 70.976, "count": 13, "max_ms": 81.873, "p95_ms": 81.873}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.957, "count": 13, "max_ms": 97.055, "p95_ms": 97.055}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.046, "count": 13, "max_ms": 109.397, "p95_ms": 109.397}, "read_interval": {"avg_ms": 1974.522, "count": 13, "max_ms": 24357.881, "p95_ms": 24357.881}, "triton_wait": {"avg_ms": 85.467, "count": 13, "max_ms": 97.067, "p95_ms": 97.067}, "visualize": {"avg_ms": 3.613, "count": 13, "max_ms": 3.765, "p95_ms": 3.765}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 6.176, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1786}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5994.539, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bd03ae4800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bd03ae4800] error while decoding MB 0 44, bytestream 5140 [h264 @ 0x55bd036da100] cabac decode of qscale diff failed at 44 33 [h264 @ 0x55bd036da100] error while decoding MB 44 33, bytestream 14620 [h264 @ 0x55bd03a15980] cabac decode of qscale diff failed at 53 5 [h264 @ 0x55bd03a15980] error while decoding MB 53 5, bytestream 629 [in#0/rtsp @ 0x55bd02d27e00] Error during demuxing: Connection timed out [in", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 346, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:34:24+09:00", "last_error_summary": "[h264 @ 0x55bd03ae4800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bd03ae4800] error while decoding MB 0 44, bytestream 5140 [h264 @ 0x55bd036da100] cabac decode of qscale diff failed at 44 33 [h264 @ 0x55bd036da100] error while decoding MB 44 33, bytestream 14620 [h264 @ 0x55bd03a15980] cabac decode of qscale diff failed at 53 5 [h264 @ 0x55bd03a15980] error while decoding MB 53 5, bytestream 629 [in#0/rtsp @ 0x55bd02d27e00] Error during demuxing: Connection timed out [in", "last_exit_at": "2026-07-06T20:31:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.619, "last_started_at": "2026-07-06T20:31:07+09:00", "pid": 26421, "process": {"exists": false, "pid": 26421}, "restart_count": 116, "restart_count_per_hour": 20, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 117}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.672, "sample_count": 2210, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 214333, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:34:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-06T20:34:24+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T20:34:24+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T20:34:24.455196+09:00", "mtime_epoch": 1783337664.4551964, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129014}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:34:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1640.125, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.305, "pushed": 3.305, "read": 3.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.517, "count": 99, "max_ms": 5.56, "p95_ms": 4.362}, "gstreamer_push": {"avg_ms": 1.299, "count": 99, "max_ms": 1.555, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.317, "count": 99, "max_ms": 81.379, "p95_ms": 79.676}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.555, "count": 99, "max_ms": 113.742, "p95_ms": 101.904}, "yolo11_640_fire_11class": {"avg_ms": 73.977, "count": 99, "max_ms": 88.074, "p95_ms": 84.222}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.275, "count": 99, "max_ms": 102.178, "p95_ms": 95.517}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.815, "count": 99, "max_ms": 141.74, "p95_ms": 114.964}, "read_interval": {"avg_ms": 302.987, "count": 99, "max_ms": 10129.739, "p95_ms": 137.269}, "triton_wait": {"avg_ms": 86.462, "count": 99, "max_ms": 113.754, "p95_ms": 102.19}, "visualize": {"avg_ms": 3.57, "count": 99, "max_ms": 4.989, "p95_ms": 4.088}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1786}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.996, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:34:24+09:00", "last_error_summary": "[h264 @ 0x55bd03ae4800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bd03ae4800] error while decoding MB 0 44, bytestream 5140 [h264 @ 0x55bd036da100] cabac decode of qscale diff failed at 44 33 [h264 @ 0x55bd036da100] error while decoding MB 44 33, bytestream 14620 [h264 @ 0x55bd03a15980] cabac decode of qscale diff failed at 53 5 [h264 @ 0x55bd03a15980] error while decoding MB 53 5, bytestream 629 [in#0/rtsp @ 0x55bd02d27e00] Error during demuxing: Connection timed out [in", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 346, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:34:24+09:00", "last_error_summary": "[h264 @ 0x55bd03ae4800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bd03ae4800] error while decoding MB 0 44, bytestream 5140 [h264 @ 0x55bd036da100] cabac decode of qscale diff failed at 44 33 [h264 @ 0x55bd036da100] error while decoding MB 44 33, bytestream 14620 [h264 @ 0x55bd03a15980] cabac decode of qscale diff failed at 53 5 [h264 @ 0x55bd03a15980] error while decoding MB 53 5, bytestream 629 [in#0/rtsp @ 0x55bd02d27e00] Error during demuxing: Connection timed out [in", "last_exit_at": "2026-07-06T20:34:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.677, "last_started_at": "2026-07-06T20:31:07+09:00", "pid": 26421, "process": {"exists": false, "pid": 26421}, "restart_count": 116, "restart_count_per_hour": 20, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 117}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.672, "sample_count": 2210, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 214334, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:34:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T20:34:24+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T20:34:24+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T20:34:24.455196+09:00", "mtime_epoch": 1783337664.4551964, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129014}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:34:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1640.125, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.3, "pushed": 3.3, "read": 3.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.515, "count": 99, "max_ms": 5.56, "p95_ms": 4.362}, "gstreamer_push": {"avg_ms": 1.301, "count": 99, "max_ms": 1.555, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.395, "count": 99, "max_ms": 81.379, "p95_ms": 79.676}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.649, "count": 99, "max_ms": 113.742, "p95_ms": 101.904}, "yolo11_640_fire_11class": {"avg_ms": 74.139, "count": 99, "max_ms": 88.074, "p95_ms": 84.222}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.363, "count": 99, "max_ms": 102.178, "p95_ms": 95.517}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.905, "count": 99, "max_ms": 141.74, "p95_ms": 114.964}, "read_interval": {"avg_ms": 302.987, "count": 99, "max_ms": 10129.739, "p95_ms": 137.269}, "triton_wait": {"avg_ms": 86.55, "count": 99, "max_ms": 113.754, "p95_ms": 102.19}, "visualize": {"avg_ms": 3.569, "count": 99, "max_ms": 4.989, "p95_ms": 4.088}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1786}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.719, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 346, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:34:24+09:00", "last_error_summary": "[h264 @ 0x55bd03ae4800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bd03ae4800] error while decoding MB 0 44, bytestream 5140 [h264 @ 0x55bd036da100] cabac decode of qscale diff failed at 44 33 [h264 @ 0x55bd036da100] error while decoding MB 44 33, bytestream 14620 [h264 @ 0x55bd03a15980] cabac decode of qscale diff failed at 53 5 [h264 @ 0x55bd03a15980] error while decoding MB 53 5, bytestream 629 [in#0/rtsp @ 0x55bd02d27e00] Error during demuxing: Connection timed out [in", "last_exit_at": "2026-07-06T20:34:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.685, "last_started_at": "2026-07-06T20:31:07+09:00", "pid": 26421, "process": {"exists": false, "pid": 26421}, "restart_count": 116, "restart_count_per_hour": 20, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 117}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.672, "sample_count": 2210, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 214334, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:34:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T20:34:24+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T20:34:24+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T20:34:24.455196+09:00", "mtime_epoch": 1783337664.4551964, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129014}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:34:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1640.125, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.31, "pushed": 3.31, "read": 3.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.515, "count": 99, "max_ms": 5.56, "p95_ms": 4.362}, "gstreamer_push": {"avg_ms": 1.301, "count": 99, "max_ms": 1.555, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.395, "count": 99, "max_ms": 81.379, "p95_ms": 79.676}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.649, "count": 99, "max_ms": 113.742, "p95_ms": 101.904}, "yolo11_640_fire_11class": {"avg_ms": 74.139, "count": 99, "max_ms": 88.074, "p95_ms": 84.222}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.363, "count": 99, "max_ms": 102.178, "p95_ms": 95.517}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.905, "count": 99, "max_ms": 141.74, "p95_ms": 114.964}, "read_interval": {"avg_ms": 304.968, "count": 98, "max_ms": 10129.739, "p95_ms": 137.269}, "triton_wait": {"avg_ms": 86.55, "count": 99, "max_ms": 113.754, "p95_ms": 102.19}, "visualize": {"avg_ms": 3.569, "count": 99, "max_ms": 4.989, "p95_ms": 4.088}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1786}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6101.887, "limit_mb": 8796093022207.996}, "detail": {"pid": 26680, "restart_count": 117, "start_count": 118}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 346, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:34:24+09:00", "last_error_summary": "[h264 @ 0x55bd03ae4800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bd03ae4800] error while decoding MB 0 44, bytestream 5140 [h264 @ 0x55bd036da100] cabac decode of qscale diff failed at 44 33 [h264 @ 0x55bd036da100] error while decoding MB 44 33, bytestream 14620 [h264 @ 0x55bd03a15980] cabac decode of qscale diff failed at 53 5 [h264 @ 0x55bd03a15980] error while decoding MB 53 5, bytestream 629 [in#0/rtsp @ 0x55bd02d27e00] Error during demuxing: Connection timed out [in", "last_exit_at": "2026-07-06T20:34:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:34:27+09:00", "pid": 26680, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.234, "name": "ffmpeg", "pid": 26680, "rss_mb": 138.234, "state": "S (sleeping)", "threads": 106}, "restart_count": 117, "restart_count_per_hour": 21, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 118}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.392, "sample_count": 2175, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 214334, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:34:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.7, "last_frame_pushed_at": "2026-07-06T20:34:24+09:00", "last_frame_read_age_sec": 4.803, "last_frame_read_at": "2026-07-06T20:34:24+09:00", "mjpeg": {"age_sec": 4.685, "exists": true, "mtime": "2026-07-06T20:34:24.571198+09:00", "mtime_epoch": 1783337664.5711985, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129197}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:34:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1640.125, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 57, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.9, "pushed": 1.9, "read": 1.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.523, "count": 57, "max_ms": 5.56, "p95_ms": 3.929}, "gstreamer_push": {"avg_ms": 1.317, "count": 57, "max_ms": 1.555, "p95_ms": 1.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.434, "count": 57, "max_ms": 81.379, "p95_ms": 78.659}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.383, "count": 57, "max_ms": 112.028, "p95_ms": 101.904}, "yolo11_640_fire_11class": {"avg_ms": 73.665, "count": 57, "max_ms": 88.074, "p95_ms": 84.774}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.319, "count": 57, "max_ms": 102.178, "p95_ms": 96.001}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.727, "count": 57, "max_ms": 141.74, "p95_ms": 121.51}, "read_interval": {"avg_ms": 443.866, "count": 57, "max_ms": 10129.739, "p95_ms": 151.019}, "triton_wait": {"avg_ms": 85.972, "count": 57, "max_ms": 112.041, "p95_ms": 102.19}, "visualize": {"avg_ms": 3.624, "count": 57, "max_ms": 4.989, "p95_ms": 4.122}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5937.988, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562d2032fd00] error while decoding MB 39 50, bytestream -5 [in#0/rtsp @ 0x562d20303e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d20303e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:37:06+09:00", "last_error_summary": "[h264 @ 0x562d2032fd00] error while decoding MB 39 50, bytestream -5 [in#0/rtsp @ 0x562d20303e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d20303e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:32:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 278.677, "last_started_at": "2026-07-06T20:32:27+09:00", "pid": 26528, "process": {"exists": false, "pid": 26528}, "restart_count": 47, "restart_count_per_hour": 1, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.883, "sample_count": 2349, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245072, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:37:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T20:37:06+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T20:37:06+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T20:37:06.082100+09:00", "mtime_epoch": 1783337826.0820997, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 112155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:37:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1528.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.75, "pushed": 0.75, "read": 0.808}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.185, "count": 13, "max_ms": 5.926, "p95_ms": 5.926}, "gstreamer_push": {"avg_ms": 1.287, "count": 13, "max_ms": 1.399, "p95_ms": 1.399}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.552, "count": 13, "max_ms": 82.024, "p95_ms": 82.024}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.641, "count": 13, "max_ms": 110.135, "p95_ms": 110.135}, "yolo11_640_fire_11class": {"avg_ms": 66.963, "count": 13, "max_ms": 83.732, "p95_ms": 83.732}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.423, "count": 13, "max_ms": 92.38, "p95_ms": 92.38}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.883, "count": 13, "max_ms": 120.851, "p95_ms": 120.851}, "read_interval": {"avg_ms": 2323.853, "count": 14, "max_ms": 16079.44, "p95_ms": 16079.44}, "triton_wait": {"avg_ms": 76.069, "count": 13, "max_ms": 110.144, "p95_ms": 110.144}, "visualize": {"avg_ms": 3.642, "count": 13, "max_ms": 4.526, "p95_ms": 4.526}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.33, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5937.363, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:37:06+09:00", "last_error_summary": "[h264 @ 0x562d2032fd00] error while decoding MB 39 50, bytestream -5 [in#0/rtsp @ 0x562d20303e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d20303e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:37:06+09:00", "last_error_summary": "[h264 @ 0x562d2032fd00] error while decoding MB 39 50, bytestream -5 [in#0/rtsp @ 0x562d20303e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d20303e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:37:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 278.748, "last_started_at": "2026-07-06T20:32:27+09:00", "pid": 26528, "process": {"exists": false, "pid": 26528}, "restart_count": 47, "restart_count_per_hour": 1, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.992, "sample_count": 2350, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245073, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:37:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T20:37:06+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-06T20:37:06+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T20:37:06.082100+09:00", "mtime_epoch": 1783337826.0820997, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 112155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:37:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1528.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.805, "pushed": 0.805, "read": 0.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.178, "count": 14, "max_ms": 5.926, "p95_ms": 5.926}, "gstreamer_push": {"avg_ms": 1.286, "count": 14, "max_ms": 1.399, "p95_ms": 1.399}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.233, "count": 14, "max_ms": 82.077, "p95_ms": 82.077}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.407, "count": 14, "max_ms": 111.368, "p95_ms": 111.368}, "yolo11_640_fire_11class": {"avg_ms": 68.358, "count": 14, "max_ms": 86.488, "p95_ms": 86.488}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.001, "count": 14, "max_ms": 95.51, "p95_ms": 95.51}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.405, "count": 14, "max_ms": 124.198, "p95_ms": 124.198}, "read_interval": {"avg_ms": 2323.853, "count": 14, "max_ms": 16079.44, "p95_ms": 16079.44}, "triton_wait": {"avg_ms": 78.591, "count": 14, "max_ms": 111.381, "p95_ms": 111.381}, "visualize": {"avg_ms": 3.65, "count": 14, "max_ms": 4.526, "p95_ms": 4.526}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.401, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5937.363, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:37:06+09:00", "last_error_summary": "[h264 @ 0x562d2032fd00] error while decoding MB 39 50, bytestream -5 [in#0/rtsp @ 0x562d20303e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d20303e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:37:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 278.755, "last_started_at": "2026-07-06T20:32:27+09:00", "pid": 26528, "process": {"exists": false, "pid": 26528}, "restart_count": 47, "restart_count_per_hour": 1, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.992, "sample_count": 2350, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245073, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:37:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T20:37:06+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-06T20:37:06+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-06T20:37:06.082100+09:00", "mtime_epoch": 1783337826.0820997, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 112155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:37:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1528.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.804, "pushed": 0.804, "read": 0.804}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.178, "count": 14, "max_ms": 5.926, "p95_ms": 5.926}, "gstreamer_push": {"avg_ms": 1.286, "count": 14, "max_ms": 1.399, "p95_ms": 1.399}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.233, "count": 14, "max_ms": 82.077, "p95_ms": 82.077}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.407, "count": 14, "max_ms": 111.368, "p95_ms": 111.368}, "yolo11_640_fire_11class": {"avg_ms": 68.358, "count": 14, "max_ms": 86.488, "p95_ms": 86.488}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.001, "count": 14, "max_ms": 95.51, "p95_ms": 95.51}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.405, "count": 14, "max_ms": 124.198, "p95_ms": 124.198}, "read_interval": {"avg_ms": 2323.853, "count": 14, "max_ms": 16079.44, "p95_ms": 16079.44}, "triton_wait": {"avg_ms": 78.591, "count": 14, "max_ms": 111.381, "p95_ms": 111.381}, "visualize": {"avg_ms": 3.65, "count": 14, "max_ms": 4.526, "p95_ms": 4.526}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.408, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2042}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6066.379, "limit_mb": 8796093022207.996}, "detail": {"pid": 26787, "restart_count": 48, "start_count": 49}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:37:06+09:00", "last_error_summary": "[h264 @ 0x562d2032fd00] error while decoding MB 39 50, bytestream -5 [in#0/rtsp @ 0x562d20303e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d20303e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:37:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:37:09+09:00", "pid": 26787, "process": {"exists": true, "fd_count": 5, "hwm_mb": 157.027, "name": "ffmpeg", "pid": 26787, "rss_mb": 156.863, "state": "R (running)", "threads": 49}, "restart_count": 48, "restart_count_per_hour": 2, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.992, "sample_count": 2350, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245073, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:37:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.904, "last_frame_pushed_at": "2026-07-06T20:37:06+09:00", "last_frame_read_age_sec": 5.031, "last_frame_read_at": "2026-07-06T20:37:06+09:00", "mjpeg": {"age_sec": 4.889, "exists": true, "mtime": "2026-07-06T20:37:06.222102+09:00", "mtime_epoch": 1783337826.2221024, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 113514}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:37:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1528.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.628, "pushed": 0.628, "read": 0.628}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.178, "count": 14, "max_ms": 5.926, "p95_ms": 5.926}, "gstreamer_push": {"avg_ms": 1.286, "count": 14, "max_ms": 1.399, "p95_ms": 1.399}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.233, "count": 14, "max_ms": 82.077, "p95_ms": 82.077}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.407, "count": 14, "max_ms": 111.368, "p95_ms": 111.368}, "yolo11_640_fire_11class": {"avg_ms": 68.358, "count": 14, "max_ms": 86.488, "p95_ms": 86.488}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.001, "count": 14, "max_ms": 95.51, "p95_ms": 95.51}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.405, "count": 14, "max_ms": 124.198, "p95_ms": 124.198}, "read_interval": {"avg_ms": 2323.853, "count": 14, "max_ms": 16079.44, "p95_ms": 16079.44}, "triton_wait": {"avg_ms": 78.591, "count": 14, "max_ms": 111.381, "p95_ms": 111.381}, "visualize": {"avg_ms": 3.65, "count": 14, "max_ms": 4.526, "p95_ms": 4.526}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.3, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1812}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6009.789, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561b055dc200] error while decoding MB 8 26, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 108, "h264_decode_error_count": 347, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:40:38+09:00", "last_error_summary": "[h264 @ 0x561b055dc200] error while decoding MB 8 26, bytestream -9", "last_exit_at": "2026-07-06T20:34:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 371.293, "last_started_at": "2026-07-06T20:34:27+09:00", "pid": 26680, "process": {"exists": false, "pid": 26680}, "restart_count": 117, "restart_count_per_hour": 19, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 118}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.867, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 217525, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:40:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T20:40:38+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-06T20:40:38+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T20:40:38.497908+09:00", "mtime_epoch": 1783338038.497908, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148512}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:40:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1641.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 243, "pushed_frames": 243, "read_frames": 244, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.101, "pushed": 8.101, "read": 8.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.297, "count": 243, "max_ms": 9.55, "p95_ms": 7.642}, "gstreamer_push": {"avg_ms": 1.189, "count": 243, "max_ms": 2.212, "p95_ms": 1.409}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.359, "count": 243, "max_ms": 88.779, "p95_ms": 79.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.316, "count": 243, "max_ms": 117.262, "p95_ms": 93.578}, "yolo11_640_fire_11class": {"avg_ms": 63.626, "count": 243, "max_ms": 89.49, "p95_ms": 82.859}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.034, "count": 243, "max_ms": 103.665, "p95_ms": 98.34}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.025, "count": 243, "max_ms": 182.261, "p95_ms": 158.812}, "read_interval": {"avg_ms": 123.208, "count": 244, "max_ms": 193.113, "p95_ms": 168.186}, "triton_wait": {"avg_ms": 73.944, "count": 243, "max_ms": 117.275, "p95_ms": 100.939}, "visualize": {"avg_ms": 4.687, "count": 243, "max_ms": 8.248, "p95_ms": 6.421}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1812}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6009.73, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:40:38+09:00", "last_error_summary": "[h264 @ 0x561b055dc200] error while decoding MB 8 26, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 108, "h264_decode_error_count": 347, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:40:38+09:00", "last_error_summary": "[h264 @ 0x561b055dc200] error while decoding MB 8 26, bytestream -9", "last_exit_at": "2026-07-06T20:40:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 371.319, "last_started_at": "2026-07-06T20:34:27+09:00", "pid": 26680, "process": {"exists": false, "pid": 26680}, "restart_count": 117, "restart_count_per_hour": 19, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 118}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.867, "sample_count": 2565, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 217526, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:40:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T20:40:38+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-06T20:40:38+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-06T20:40:38.497908+09:00", "mtime_epoch": 1783338038.497908, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148512}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:40:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1641.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 244, "pushed_frames": 244, "read_frames": 243, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.159, "pushed": 8.159, "read": 8.126}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.287, "count": 244, "max_ms": 9.55, "p95_ms": 7.642}, "gstreamer_push": {"avg_ms": 1.188, "count": 244, "max_ms": 2.212, "p95_ms": 1.409}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.343, "count": 244, "max_ms": 88.779, "p95_ms": 79.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.262, "count": 244, "max_ms": 117.262, "p95_ms": 93.578}, "yolo11_640_fire_11class": {"avg_ms": 63.587, "count": 244, "max_ms": 89.49, "p95_ms": 82.859}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.999, "count": 244, "max_ms": 103.665, "p95_ms": 98.34}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.883, "count": 244, "max_ms": 182.261, "p95_ms": 158.812}, "read_interval": {"avg_ms": 123.195, "count": 243, "max_ms": 193.113, "p95_ms": 168.186}, "triton_wait": {"avg_ms": 73.901, "count": 244, "max_ms": 117.275, "p95_ms": 100.939}, "visualize": {"avg_ms": 4.694, "count": 244, "max_ms": 8.248, "p95_ms": 6.421}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1812}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6009.715, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 108, "h264_decode_error_count": 347, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:40:38+09:00", "last_error_summary": "[h264 @ 0x561b055dc200] error while decoding MB 8 26, bytestream -9", "last_exit_at": "2026-07-06T20:40:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 371.326, "last_started_at": "2026-07-06T20:34:27+09:00", "pid": 26680, "process": {"exists": false, "pid": 26680}, "restart_count": 117, "restart_count_per_hour": 19, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 118}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.867, "sample_count": 2565, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 217526, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:40:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T20:40:38+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-06T20:40:38+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-06T20:40:38.497908+09:00", "mtime_epoch": 1783338038.497908, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148512}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:40:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1641.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 244, "pushed_frames": 244, "read_frames": 243, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.157, "pushed": 8.157, "read": 8.124}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.287, "count": 244, "max_ms": 9.55, "p95_ms": 7.642}, "gstreamer_push": {"avg_ms": 1.188, "count": 244, "max_ms": 2.212, "p95_ms": 1.409}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.343, "count": 244, "max_ms": 88.779, "p95_ms": 79.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.262, "count": 244, "max_ms": 117.262, "p95_ms": 93.578}, "yolo11_640_fire_11class": {"avg_ms": 63.587, "count": 244, "max_ms": 89.49, "p95_ms": 82.859}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.999, "count": 244, "max_ms": 103.665, "p95_ms": 98.34}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.883, "count": 244, "max_ms": 182.261, "p95_ms": 158.812}, "read_interval": {"avg_ms": 123.195, "count": 243, "max_ms": 193.113, "p95_ms": 168.186}, "triton_wait": {"avg_ms": 73.901, "count": 244, "max_ms": 117.275, "p95_ms": 100.939}, "visualize": {"avg_ms": 4.694, "count": 244, "max_ms": 8.248, "p95_ms": 6.421}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1812}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6117.449, "limit_mb": 8796093022207.996}, "detail": {"pid": 26939, "restart_count": 118, "start_count": 119}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 108, "h264_decode_error_count": 347, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:40:38+09:00", "last_error_summary": "[h264 @ 0x561b055dc200] error while decoding MB 8 26, bytestream -9", "last_exit_at": "2026-07-06T20:40:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:40:40+09:00", "pid": 26939, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.676, "name": "ffmpeg", "pid": 26939, "rss_mb": 141.676, "state": "S (sleeping)", "threads": 106}, "restart_count": 118, "restart_count_per_hour": 20, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 119}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.079, "sample_count": 2527, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 217526, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:40:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.353, "last_frame_pushed_at": "2026-07-06T20:40:38+09:00", "last_frame_read_age_sec": 4.432, "last_frame_read_at": "2026-07-06T20:40:38+09:00", "mjpeg": {"age_sec": 4.337, "exists": true, "mtime": "2026-07-06T20:40:38.585910+09:00", "mtime_epoch": 1783338038.5859096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 146875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:40:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1641.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 210, "pushed_frames": 210, "read_frames": 209, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.022, "pushed": 7.022, "read": 6.989}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.267, "count": 210, "max_ms": 9.55, "p95_ms": 7.658}, "gstreamer_push": {"avg_ms": 1.191, "count": 210, "max_ms": 2.212, "p95_ms": 1.412}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.042, "count": 210, "max_ms": 85.201, "p95_ms": 79.323}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.681, "count": 210, "max_ms": 117.262, "p95_ms": 91.778}, "yolo11_640_fire_11class": {"avg_ms": 63.34, "count": 210, "max_ms": 87.962, "p95_ms": 82.42}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.752, "count": 210, "max_ms": 103.665, "p95_ms": 97.239}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.972, "count": 210, "max_ms": 182.261, "p95_ms": 157.527}, "read_interval": {"avg_ms": 122.339, "count": 209, "max_ms": 193.113, "p95_ms": 167.28}, "triton_wait": {"avg_ms": 73.423, "count": 210, "max_ms": 117.275, "p95_ms": 100.497}, "visualize": {"avg_ms": 4.675, "count": 210, "max_ms": 8.248, "p95_ms": 6.406}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1834}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6010.973, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55efbc305240] error while decoding MB 46 10, bytestream -24", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 348, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:45:55+09:00", "last_error_summary": "[h264 @ 0x55efbc305240] error while decoding MB 46 10, bytestream -24", "last_exit_at": "2026-07-06T20:40:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 314.29, "last_started_at": "2026-07-06T20:40:40+09:00", "pid": 26939, "process": {"exists": false, "pid": 26939}, "restart_count": 118, "restart_count_per_hour": 18, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 119}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.525, "sample_count": 2525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220154, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:45:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-06T20:45:55+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-06T20:45:55+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-06T20:45:55.163575+09:00", "mtime_epoch": 1783338355.1635754, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:45:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1642.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 262, "pushed_frames": 262, "read_frames": 262, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.738, "pushed": 8.738, "read": 8.738}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.398, "count": 262, "max_ms": 8.755, "p95_ms": 6.25}, "gstreamer_push": {"avg_ms": 1.267, "count": 262, "max_ms": 1.984, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.008, "count": 262, "max_ms": 88.473, "p95_ms": 79.484}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.467, "count": 262, "max_ms": 114.242, "p95_ms": 94.164}, "yolo11_640_fire_11class": {"avg_ms": 66.877, "count": 262, "max_ms": 86.649, "p95_ms": 81.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.847, "count": 262, "max_ms": 101.583, "p95_ms": 93.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.368, "count": 262, "max_ms": 163.958, "p95_ms": 142.212}, "read_interval": {"avg_ms": 114.631, "count": 262, "max_ms": 176.821, "p95_ms": 153.661}, "triton_wait": {"avg_ms": 77.354, "count": 262, "max_ms": 114.253, "p95_ms": 98.884}, "visualize": {"avg_ms": 4.157, "count": 262, "max_ms": 6.357, "p95_ms": 5.286}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1834}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6010.434, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:45:55+09:00", "last_error_summary": "[h264 @ 0x55efbc305240] error while decoding MB 46 10, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 348, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:45:55+09:00", "last_error_summary": "[h264 @ 0x55efbc305240] error while decoding MB 46 10, bytestream -24", "last_exit_at": "2026-07-06T20:45:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 314.345, "last_started_at": "2026-07-06T20:40:40+09:00", "pid": 26939, "process": {"exists": false, "pid": 26939}, "restart_count": 118, "restart_count_per_hour": 18, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 119}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.525, "sample_count": 2525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220155, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:45:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T20:45:55+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-06T20:45:55+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-06T20:45:55.163575+09:00", "mtime_epoch": 1783338355.1635754, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:45:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1642.961, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 262, "pushed_frames": 262, "read_frames": 261, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.76, "pushed": 8.76, "read": 8.727}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.393, "count": 262, "max_ms": 8.755, "p95_ms": 6.25}, "gstreamer_push": {"avg_ms": 1.267, "count": 262, "max_ms": 1.984, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.039, "count": 262, "max_ms": 88.473, "p95_ms": 79.484}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.486, "count": 262, "max_ms": 114.242, "p95_ms": 94.164}, "yolo11_640_fire_11class": {"avg_ms": 66.872, "count": 262, "max_ms": 86.649, "p95_ms": 81.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.845, "count": 262, "max_ms": 101.583, "p95_ms": 93.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.359, "count": 262, "max_ms": 163.958, "p95_ms": 142.212}, "read_interval": {"avg_ms": 114.635, "count": 261, "max_ms": 176.821, "p95_ms": 153.661}, "triton_wait": {"avg_ms": 77.352, "count": 262, "max_ms": 114.253, "p95_ms": 98.884}, "visualize": {"avg_ms": 4.155, "count": 262, "max_ms": 6.357, "p95_ms": 5.286}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1834}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6010.516, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 348, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:45:55+09:00", "last_error_summary": "[h264 @ 0x55efbc305240] error while decoding MB 46 10, bytestream -24", "last_exit_at": "2026-07-06T20:45:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 314.352, "last_started_at": "2026-07-06T20:40:40+09:00", "pid": 26939, "process": {"exists": false, "pid": 26939}, "restart_count": 118, "restart_count_per_hour": 18, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 119}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.525, "sample_count": 2525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220155, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:45:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T20:45:55+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T20:45:55+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-06T20:45:55.163575+09:00", "mtime_epoch": 1783338355.1635754, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:45:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1642.961, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 262, "pushed_frames": 262, "read_frames": 261, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.758, "pushed": 8.758, "read": 8.725}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.393, "count": 262, "max_ms": 8.755, "p95_ms": 6.25}, "gstreamer_push": {"avg_ms": 1.267, "count": 262, "max_ms": 1.984, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.039, "count": 262, "max_ms": 88.473, "p95_ms": 79.484}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.486, "count": 262, "max_ms": 114.242, "p95_ms": 94.164}, "yolo11_640_fire_11class": {"avg_ms": 66.872, "count": 262, "max_ms": 86.649, "p95_ms": 81.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.845, "count": 262, "max_ms": 101.583, "p95_ms": 93.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.359, "count": 262, "max_ms": 163.958, "p95_ms": 142.212}, "read_interval": {"avg_ms": 114.635, "count": 261, "max_ms": 176.821, "p95_ms": 153.661}, "triton_wait": {"avg_ms": 77.352, "count": 262, "max_ms": 114.253, "p95_ms": 98.884}, "visualize": {"avg_ms": 4.155, "count": 262, "max_ms": 6.357, "p95_ms": 5.286}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1834}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6117.988, "limit_mb": 8796093022207.996}, "detail": {"pid": 27046, "restart_count": 119, "start_count": 120}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 348, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:45:55+09:00", "last_error_summary": "[h264 @ 0x55efbc305240] error while decoding MB 46 10, bytestream -24", "last_exit_at": "2026-07-06T20:45:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:45:57+09:00", "pid": 27046, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.707, "name": "ffmpeg", "pid": 27046, "rss_mb": 138.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 119, "restart_count_per_hour": 19, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 120}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.525, "sample_count": 2485, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220155, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:45:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.624, "last_frame_pushed_at": "2026-07-06T20:45:55+09:00", "last_frame_read_age_sec": 4.724, "last_frame_read_at": "2026-07-06T20:45:55+09:00", "mjpeg": {"age_sec": 4.608, "exists": true, "mtime": "2026-07-06T20:45:55.275577+09:00", "mtime_epoch": 1783338355.2755773, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123499}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:45:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1642.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.457, "pushed": 7.457, "read": 7.423}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.314, "count": 223, "max_ms": 8.755, "p95_ms": 6.358}, "gstreamer_push": {"avg_ms": 1.273, "count": 223, "max_ms": 1.984, "p95_ms": 1.444}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.871, "count": 223, "max_ms": 88.473, "p95_ms": 79.484}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.565, "count": 223, "max_ms": 114.242, "p95_ms": 92.052}, "yolo11_640_fire_11class": {"avg_ms": 67.336, "count": 223, "max_ms": 86.649, "p95_ms": 81.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.488, "count": 223, "max_ms": 101.583, "p95_ms": 93.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.492, "count": 223, "max_ms": 163.958, "p95_ms": 141.937}, "read_interval": {"avg_ms": 113.909, "count": 222, "max_ms": 176.821, "p95_ms": 152.451}, "triton_wait": {"avg_ms": 77.733, "count": 223, "max_ms": 114.253, "p95_ms": 97.724}, "visualize": {"avg_ms": 4.105, "count": 223, "max_ms": 6.337, "p95_ms": 5.286}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5959.031, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562d9a223340] error while decoding MB 71 14, bytestream -5 [in#0/rtsp @ 0x562d99538e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d99538e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:49:47+09:00", "last_error_summary": "[h264 @ 0x562d9a223340] error while decoding MB 71 14, bytestream -5 [in#0/rtsp @ 0x562d99538e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d99538e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:37:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 758.515, "last_started_at": "2026-07-06T20:37:09+09:00", "pid": 26787, "process": {"exists": false, "pid": 26787}, "restart_count": 48, "restart_count_per_hour": 2, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.677, "sample_count": 2575, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 252217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:49:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-06T20:49:47+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-06T20:49:47+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-06T20:49:47.559730+09:00", "mtime_epoch": 1783338587.5597296, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106562}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:49:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1531.902, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 82, "pushed_frames": 82, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.734, "pushed": 2.734, "read": 2.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.127, "count": 82, "max_ms": 6.363, "p95_ms": 4.741}, "gstreamer_push": {"avg_ms": 1.254, "count": 82, "max_ms": 1.458, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.973, "count": 82, "max_ms": 91.001, "p95_ms": 80.568}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.027, "count": 82, "max_ms": 100.12, "p95_ms": 92.969}, "yolo11_640_fire_11class": {"avg_ms": 74.698, "count": 82, "max_ms": 90.051, "p95_ms": 82.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.334, "count": 82, "max_ms": 103.128, "p95_ms": 94.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.71, "count": 82, "max_ms": 115.912, "p95_ms": 110.292}, "read_interval": {"avg_ms": 361.938, "count": 83, "max_ms": 12017.354, "p95_ms": 128.554}, "triton_wait": {"avg_ms": 85.096, "count": 82, "max_ms": 103.142, "p95_ms": 96.472}, "visualize": {"avg_ms": 3.564, "count": 82, "max_ms": 4.896, "p95_ms": 3.989}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5958.586, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:49:47+09:00", "last_error_summary": "[h264 @ 0x562d9a223340] error while decoding MB 71 14, bytestream -5 [in#0/rtsp @ 0x562d99538e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d99538e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:49:47+09:00", "last_error_summary": "[h264 @ 0x562d9a223340] error while decoding MB 71 14, bytestream -5 [in#0/rtsp @ 0x562d99538e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d99538e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:49:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 758.55, "last_started_at": "2026-07-06T20:37:09+09:00", "pid": 26787, "process": {"exists": false, "pid": 26787}, "restart_count": 48, "restart_count_per_hour": 2, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.677, "sample_count": 2576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 252218, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:49:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T20:49:47+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-06T20:49:47+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T20:49:47.559730+09:00", "mtime_epoch": 1783338587.5597296, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106562}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:49:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1531.902, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.773, "pushed": 2.773, "read": 2.74}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.137, "count": 83, "max_ms": 6.363, "p95_ms": 4.786}, "gstreamer_push": {"avg_ms": 1.251, "count": 83, "max_ms": 1.458, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.024, "count": 83, "max_ms": 91.001, "p95_ms": 80.568}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.039, "count": 83, "max_ms": 100.12, "p95_ms": 92.969}, "yolo11_640_fire_11class": {"avg_ms": 74.704, "count": 83, "max_ms": 90.051, "p95_ms": 82.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.275, "count": 83, "max_ms": 103.128, "p95_ms": 94.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.659, "count": 83, "max_ms": 115.912, "p95_ms": 110.292}, "read_interval": {"avg_ms": 365.006, "count": 82, "max_ms": 12017.354, "p95_ms": 128.554}, "triton_wait": {"avg_ms": 85.036, "count": 83, "max_ms": 103.142, "p95_ms": 96.472}, "visualize": {"avg_ms": 3.565, "count": 83, "max_ms": 4.896, "p95_ms": 3.989}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5958.32, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:49:47+09:00", "last_error_summary": "[h264 @ 0x562d9a223340] error while decoding MB 71 14, bytestream -5 [in#0/rtsp @ 0x562d99538e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d99538e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:49:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 758.559, "last_started_at": "2026-07-06T20:37:09+09:00", "pid": 26787, "process": {"exists": false, "pid": 26787}, "restart_count": 48, "restart_count_per_hour": 2, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.677, "sample_count": 2576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 252218, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:49:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T20:49:47+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-06T20:49:47+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T20:49:47.559730+09:00", "mtime_epoch": 1783338587.5597296, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106562}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:49:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1531.902, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.773, "pushed": 2.773, "read": 2.739}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.137, "count": 83, "max_ms": 6.363, "p95_ms": 4.786}, "gstreamer_push": {"avg_ms": 1.251, "count": 83, "max_ms": 1.458, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.024, "count": 83, "max_ms": 91.001, "p95_ms": 80.568}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.039, "count": 83, "max_ms": 100.12, "p95_ms": 92.969}, "yolo11_640_fire_11class": {"avg_ms": 74.704, "count": 83, "max_ms": 90.051, "p95_ms": 82.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.275, "count": 83, "max_ms": 103.128, "p95_ms": 94.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.659, "count": 83, "max_ms": 115.912, "p95_ms": 110.292}, "read_interval": {"avg_ms": 365.006, "count": 82, "max_ms": 12017.354, "p95_ms": 128.554}, "triton_wait": {"avg_ms": 85.036, "count": 83, "max_ms": 103.142, "p95_ms": 96.472}, "visualize": {"avg_ms": 3.565, "count": 83, "max_ms": 4.896, "p95_ms": 3.989}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6145.875, "limit_mb": 8796093022207.996}, "detail": {"pid": 27153, "restart_count": 49, "start_count": 50}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:49:47+09:00", "last_error_summary": "[h264 @ 0x562d9a223340] error while decoding MB 71 14, bytestream -5 [in#0/rtsp @ 0x562d99538e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562d99538e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:49:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:49:50+09:00", "pid": 27153, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.25, "name": "ffmpeg", "pid": 27153, "rss_mb": 209.25, "state": "S (sleeping)", "threads": 106}, "restart_count": 49, "restart_count_per_hour": 3, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.707, "sample_count": 2529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 252218, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:49:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.852, "last_frame_pushed_at": "2026-07-06T20:49:47+09:00", "last_frame_read_age_sec": 4.948, "last_frame_read_at": "2026-07-06T20:49:47+09:00", "mjpeg": {"age_sec": 4.834, "exists": true, "mtime": "2026-07-06T20:49:47.671732+09:00", "mtime_epoch": 1783338587.6717317, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:49:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1531.902, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 40, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.368, "pushed": 1.368, "read": 1.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.146, "count": 41, "max_ms": 6.363, "p95_ms": 4.962}, "gstreamer_push": {"avg_ms": 1.235, "count": 41, "max_ms": 1.458, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.872, "count": 41, "max_ms": 82.223, "p95_ms": 80.129}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.798, "count": 41, "max_ms": 96.382, "p95_ms": 90.592}, "yolo11_640_fire_11class": {"avg_ms": 72.568, "count": 41, "max_ms": 84.968, "p95_ms": 82.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.712, "count": 41, "max_ms": 92.709, "p95_ms": 92.052}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.028, "count": 41, "max_ms": 110.292, "p95_ms": 105.351}, "read_interval": {"avg_ms": 628.022, "count": 40, "max_ms": 12017.354, "p95_ms": 128.554}, "triton_wait": {"avg_ms": 81.457, "count": 41, "max_ms": 96.394, "p95_ms": 92.721}, "visualize": {"avg_ms": 3.542, "count": 41, "max_ms": 4.896, "p95_ms": 4.29}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1863}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6008.102, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564f8e343fc0] top block unavailable for requested intra mode [h264 @ 0x564f8e343fc0] error while decoding MB 75 0, bytestream 1272 [h264 @ 0x564f8e3f6780] cabac decode of qscale diff failed at 29 22 [h264 @ 0x564f8e3f6780] error while decoding MB 29 22, bytestream 36988 [h264 @ 0x564f8efc7c00] cabac decode of qscale diff failed at 28 3 [h264 @ 0x564f8efc7c00] error while decoding MB 28 3, bytestream 3337 [h264 @ 0x564f8e3186c0] cabac decode of qscale diff failed at 17 21 [h264 @ 0x564f", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 110, "h264_decode_error_count": 355, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:52:38+09:00", "last_error_summary": "[h264 @ 0x564f8e343fc0] top block unavailable for requested intra mode [h264 @ 0x564f8e343fc0] error while decoding MB 75 0, bytestream 1272 [h264 @ 0x564f8e3f6780] cabac decode of qscale diff failed at 29 22 [h264 @ 0x564f8e3f6780] error while decoding MB 29 22, bytestream 36988 [h264 @ 0x564f8efc7c00] cabac decode of qscale diff failed at 28 3 [h264 @ 0x564f8efc7c00] error while decoding MB 28 3, bytestream 3337 [h264 @ 0x564f8e3186c0] cabac decode of qscale diff failed at 17 21 [h264 @ 0x564f", "last_exit_at": "2026-07-06T20:45:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 400.563, "last_started_at": "2026-07-06T20:45:57+09:00", "pid": 27046, "process": {"exists": false, "pid": 27046}, "restart_count": 119, "restart_count_per_hour": 16, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 120}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.895, "sample_count": 2579, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 223618, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:52:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T20:52:38+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-06T20:52:38+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T20:52:38.394782+09:00", "mtime_epoch": 1783338758.3947818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 128247}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:52:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1644.508, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.175, "pushed": 3.175, "read": 3.175}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.559, "count": 95, "max_ms": 5.268, "p95_ms": 4.696}, "gstreamer_push": {"avg_ms": 1.32, "count": 95, "max_ms": 1.511, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.813, "count": 95, "max_ms": 88.666, "p95_ms": 84.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.371, "count": 95, "max_ms": 111.223, "p95_ms": 91.449}, "yolo11_640_fire_11class": {"avg_ms": 72.69, "count": 95, "max_ms": 88.962, "p95_ms": 86.656}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.212, "count": 95, "max_ms": 99.707, "p95_ms": 96.048}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.501, "count": 95, "max_ms": 123.673, "p95_ms": 111.895}, "read_interval": {"avg_ms": 315.254, "count": 95, "max_ms": 10123.11, "p95_ms": 135.201}, "triton_wait": {"avg_ms": 81.253, "count": 95, "max_ms": 111.235, "p95_ms": 99.719}, "visualize": {"avg_ms": 3.643, "count": 95, "max_ms": 4.735, "p95_ms": 4.0}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1863}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6007.387, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:52:38+09:00", "last_error_summary": "[h264 @ 0x564f8e343fc0] top block unavailable for requested intra mode [h264 @ 0x564f8e343fc0] error while decoding MB 75 0, bytestream 1272 [h264 @ 0x564f8e3f6780] cabac decode of qscale diff failed at 29 22 [h264 @ 0x564f8e3f6780] error while decoding MB 29 22, bytestream 36988 [h264 @ 0x564f8efc7c00] cabac decode of qscale diff failed at 28 3 [h264 @ 0x564f8efc7c00] error while decoding MB 28 3, bytestream 3337 [h264 @ 0x564f8e3186c0] cabac decode of qscale diff failed at 17 21 [h264 @ 0x564f", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 110, "h264_decode_error_count": 355, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:52:38+09:00", "last_error_summary": "[h264 @ 0x564f8e343fc0] top block unavailable for requested intra mode [h264 @ 0x564f8e343fc0] error while decoding MB 75 0, bytestream 1272 [h264 @ 0x564f8e3f6780] cabac decode of qscale diff failed at 29 22 [h264 @ 0x564f8e3f6780] error while decoding MB 29 22, bytestream 36988 [h264 @ 0x564f8efc7c00] cabac decode of qscale diff failed at 28 3 [h264 @ 0x564f8efc7c00] error while decoding MB 28 3, bytestream 3337 [h264 @ 0x564f8e3186c0] cabac decode of qscale diff failed at 17 21 [h264 @ 0x564f", "last_exit_at": "2026-07-06T20:52:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 400.613, "last_started_at": "2026-07-06T20:45:57+09:00", "pid": 27046, "process": {"exists": false, "pid": 27046}, "restart_count": 119, "restart_count_per_hour": 16, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 120}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.895, "sample_count": 2579, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 223619, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:52:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T20:52:38+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-06T20:52:38+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T20:52:38.394782+09:00", "mtime_epoch": 1783338758.3947818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 128247}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:52:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1644.508, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.203, "pushed": 3.203, "read": 3.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.559, "count": 96, "max_ms": 5.268, "p95_ms": 4.696}, "gstreamer_push": {"avg_ms": 1.319, "count": 96, "max_ms": 1.511, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.738, "count": 96, "max_ms": 88.666, "p95_ms": 84.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.487, "count": 96, "max_ms": 111.223, "p95_ms": 91.449}, "yolo11_640_fire_11class": {"avg_ms": 72.796, "count": 96, "max_ms": 88.962, "p95_ms": 86.656}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.259, "count": 96, "max_ms": 99.707, "p95_ms": 96.048}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.552, "count": 96, "max_ms": 123.673, "p95_ms": 111.895}, "read_interval": {"avg_ms": 315.254, "count": 95, "max_ms": 10123.11, "p95_ms": 135.201}, "triton_wait": {"avg_ms": 81.307, "count": 96, "max_ms": 111.235, "p95_ms": 99.719}, "visualize": {"avg_ms": 3.641, "count": 96, "max_ms": 4.735, "p95_ms": 4.0}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1863}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6007.414, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 110, "h264_decode_error_count": 355, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:52:38+09:00", "last_error_summary": "[h264 @ 0x564f8e343fc0] top block unavailable for requested intra mode [h264 @ 0x564f8e343fc0] error while decoding MB 75 0, bytestream 1272 [h264 @ 0x564f8e3f6780] cabac decode of qscale diff failed at 29 22 [h264 @ 0x564f8e3f6780] error while decoding MB 29 22, bytestream 36988 [h264 @ 0x564f8efc7c00] cabac decode of qscale diff failed at 28 3 [h264 @ 0x564f8efc7c00] error while decoding MB 28 3, bytestream 3337 [h264 @ 0x564f8e3186c0] cabac decode of qscale diff failed at 17 21 [h264 @ 0x564f", "last_exit_at": "2026-07-06T20:52:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 400.62, "last_started_at": "2026-07-06T20:45:57+09:00", "pid": 27046, "process": {"exists": false, "pid": 27046}, "restart_count": 119, "restart_count_per_hour": 16, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 120}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.895, "sample_count": 2579, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 223619, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:52:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T20:52:38+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-06T20:52:38+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T20:52:38.394782+09:00", "mtime_epoch": 1783338758.3947818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 128247}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:52:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1644.508, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.203, "pushed": 3.203, "read": 3.169}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.559, "count": 96, "max_ms": 5.268, "p95_ms": 4.696}, "gstreamer_push": {"avg_ms": 1.319, "count": 96, "max_ms": 1.511, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.738, "count": 96, "max_ms": 88.666, "p95_ms": 84.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.487, "count": 96, "max_ms": 111.223, "p95_ms": 91.449}, "yolo11_640_fire_11class": {"avg_ms": 72.796, "count": 96, "max_ms": 88.962, "p95_ms": 86.656}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.259, "count": 96, "max_ms": 99.707, "p95_ms": 96.048}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.552, "count": 96, "max_ms": 123.673, "p95_ms": 111.895}, "read_interval": {"avg_ms": 315.254, "count": 95, "max_ms": 10123.11, "p95_ms": 135.201}, "triton_wait": {"avg_ms": 81.307, "count": 96, "max_ms": 111.235, "p95_ms": 99.719}, "visualize": {"avg_ms": 3.641, "count": 96, "max_ms": 4.735, "p95_ms": 4.0}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1863}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6123.961, "limit_mb": 8796093022207.996}, "detail": {"pid": 27305, "restart_count": 120, "start_count": 121}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 110, "h264_decode_error_count": 355, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:52:38+09:00", "last_error_summary": "[h264 @ 0x564f8e343fc0] top block unavailable for requested intra mode [h264 @ 0x564f8e343fc0] error while decoding MB 75 0, bytestream 1272 [h264 @ 0x564f8e3f6780] cabac decode of qscale diff failed at 29 22 [h264 @ 0x564f8e3f6780] error while decoding MB 29 22, bytestream 36988 [h264 @ 0x564f8efc7c00] cabac decode of qscale diff failed at 28 3 [h264 @ 0x564f8efc7c00] error while decoding MB 28 3, bytestream 3337 [h264 @ 0x564f8e3186c0] cabac decode of qscale diff failed at 17 21 [h264 @ 0x564f", "last_exit_at": "2026-07-06T20:52:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:52:40+09:00", "pid": 27305, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.105, "name": "ffmpeg", "pid": 27305, "rss_mb": 139.105, "state": "S (sleeping)", "threads": 106}, "restart_count": 120, "restart_count_per_hour": 17, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 121}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.895, "sample_count": 2541, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 223619, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:52:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.261, "last_frame_pushed_at": "2026-07-06T20:52:38+09:00", "last_frame_read_age_sec": 4.361, "last_frame_read_at": "2026-07-06T20:52:38+09:00", "mjpeg": {"age_sec": 4.243, "exists": true, "mtime": "2026-07-06T20:52:38.506784+09:00", "mtime_epoch": 1783338758.506784, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 128449}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T20:52:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1644.508, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.87, "pushed": 1.87, "read": 1.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.547, "count": 56, "max_ms": 5.268, "p95_ms": 4.884}, "gstreamer_push": {"avg_ms": 1.31, "count": 56, "max_ms": 1.511, "p95_ms": 1.452}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.17, "count": 56, "max_ms": 88.666, "p95_ms": 86.154}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.793, "count": 56, "max_ms": 107.882, "p95_ms": 91.449}, "yolo11_640_fire_11class": {"avg_ms": 72.492, "count": 56, "max_ms": 88.678, "p95_ms": 87.705}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.283, "count": 56, "max_ms": 99.707, "p95_ms": 96.972}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.539, "count": 56, "max_ms": 120.101, "p95_ms": 111.895}, "read_interval": {"avg_ms": 466.932, "count": 55, "max_ms": 10123.11, "p95_ms": 137.248}, "triton_wait": {"avg_ms": 81.38, "count": 56, "max_ms": 107.895, "p95_ms": 99.719}, "visualize": {"avg_ms": 3.617, "count": 56, "max_ms": 4.281, "p95_ms": 4.0}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2116}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5955.191, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55e6b6096200] missing picture in access unit with size 18 [h264 @ 0x55e6b6e50400] no frame! [in#0/rtsp @ 0x55e6b6091e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6b6091e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e6b6096840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:53:24+09:00", "last_error_summary": "[NULL @ 0x55e6b6096200] missing picture in access unit with size 18 [h264 @ 0x55e6b6e50400] no frame! [in#0/rtsp @ 0x55e6b6091e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6b6091e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e6b6096840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:49:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 214.32, "last_started_at": "2026-07-06T20:49:50+09:00", "pid": 27153, "process": {"exists": false, "pid": 27153}, "restart_count": 49, "restart_count_per_hour": 3, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.604, "sample_count": 2261, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 253924, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:53:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-06T20:53:24+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-06T20:53:24+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T20:53:24.783611+09:00", "mtime_epoch": 1783338804.7836106, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106380}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:53:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1532.676, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.694, "pushed": 0.694, "read": 0.752}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.069, "count": 12, "max_ms": 4.506, "p95_ms": 4.506}, "gstreamer_push": {"avg_ms": 1.282, "count": 12, "max_ms": 1.436, "p95_ms": 1.436}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.152, "count": 12, "max_ms": 76.166, "p95_ms": 76.166}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.405, "count": 12, "max_ms": 111.739, "p95_ms": 111.739}, "yolo11_640_fire_11class": {"avg_ms": 67.552, "count": 12, "max_ms": 91.698, "p95_ms": 91.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.679, "count": 12, "max_ms": 92.216, "p95_ms": 92.216}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.561, "count": 12, "max_ms": 125.334, "p95_ms": 125.334}, "read_interval": {"avg_ms": 2475.176, "count": 13, "max_ms": 16164.784, "p95_ms": 16164.784}, "triton_wait": {"avg_ms": 74.02, "count": 12, "max_ms": 111.751, "p95_ms": 111.751}, "visualize": {"avg_ms": 3.581, "count": 12, "max_ms": 3.853, "p95_ms": 3.853}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.284, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2116}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5954.215, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:53:24+09:00", "last_error_summary": "[NULL @ 0x55e6b6096200] missing picture in access unit with size 18 [h264 @ 0x55e6b6e50400] no frame! [in#0/rtsp @ 0x55e6b6091e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6b6091e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e6b6096840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:53:24+09:00", "last_error_summary": "[NULL @ 0x55e6b6096200] missing picture in access unit with size 18 [h264 @ 0x55e6b6e50400] no frame! [in#0/rtsp @ 0x55e6b6091e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6b6091e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e6b6096840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:53:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 214.394, "last_started_at": "2026-07-06T20:49:50+09:00", "pid": 27153, "process": {"exists": false, "pid": 27153}, "restart_count": 49, "restart_count_per_hour": 3, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.765, "sample_count": 2262, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 253925, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:53:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T20:53:24+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-06T20:53:24+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-06T20:53:24.783611+09:00", "mtime_epoch": 1783338804.7836106, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106380}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:53:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1532.676, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.066, "count": 13, "max_ms": 4.506, "p95_ms": 4.506}, "gstreamer_push": {"avg_ms": 1.285, "count": 13, "max_ms": 1.436, "p95_ms": 1.436}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.982, "count": 13, "max_ms": 78.94, "p95_ms": 78.94}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.227, "count": 13, "max_ms": 111.739, "p95_ms": 111.739}, "yolo11_640_fire_11class": {"avg_ms": 68.473, "count": 13, "max_ms": 91.698, "p95_ms": 91.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.197, "count": 13, "max_ms": 92.216, "p95_ms": 92.216}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.117, "count": 13, "max_ms": 125.334, "p95_ms": 125.334}, "read_interval": {"avg_ms": 2475.176, "count": 13, "max_ms": 16164.784, "p95_ms": 16164.784}, "triton_wait": {"avg_ms": 76.565, "count": 13, "max_ms": 111.751, "p95_ms": 111.751}, "visualize": {"avg_ms": 3.588, "count": 13, "max_ms": 3.853, "p95_ms": 3.853}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.358, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2116}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5954.336, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:53:24+09:00", "last_error_summary": "[NULL @ 0x55e6b6096200] missing picture in access unit with size 18 [h264 @ 0x55e6b6e50400] no frame! [in#0/rtsp @ 0x55e6b6091e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6b6091e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e6b6096840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:53:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 214.4, "last_started_at": "2026-07-06T20:49:50+09:00", "pid": 27153, "process": {"exists": false, "pid": 27153}, "restart_count": 49, "restart_count_per_hour": 3, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.765, "sample_count": 2262, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 253925, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:53:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T20:53:24+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-06T20:53:24+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-06T20:53:24.783611+09:00", "mtime_epoch": 1783338804.7836106, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106380}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:53:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1532.676, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.066, "count": 13, "max_ms": 4.506, "p95_ms": 4.506}, "gstreamer_push": {"avg_ms": 1.285, "count": 13, "max_ms": 1.436, "p95_ms": 1.436}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.982, "count": 13, "max_ms": 78.94, "p95_ms": 78.94}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.227, "count": 13, "max_ms": 111.739, "p95_ms": 111.739}, "yolo11_640_fire_11class": {"avg_ms": 68.473, "count": 13, "max_ms": 91.698, "p95_ms": 91.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.197, "count": 13, "max_ms": 92.216, "p95_ms": 92.216}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.117, "count": 13, "max_ms": 125.334, "p95_ms": 125.334}, "read_interval": {"avg_ms": 2475.176, "count": 13, "max_ms": 16164.784, "p95_ms": 16164.784}, "triton_wait": {"avg_ms": 76.565, "count": 13, "max_ms": 111.751, "p95_ms": 111.751}, "visualize": {"avg_ms": 3.588, "count": 13, "max_ms": 3.853, "p95_ms": 3.853}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.364, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2116}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5975.23, "limit_mb": 8796093022207.996}, "detail": {"pid": 27412, "restart_count": 50, "start_count": 51}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:53:24+09:00", "last_error_summary": "[NULL @ 0x55e6b6096200] missing picture in access unit with size 18 [h264 @ 0x55e6b6e50400] no frame! [in#0/rtsp @ 0x55e6b6091e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6b6091e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e6b6096840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:53:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:53:27+09:00", "pid": 27412, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.211, "name": "ffmpeg", "pid": 27412, "rss_mb": 57.211, "state": "S (sleeping)", "threads": 1}, "restart_count": 50, "restart_count_per_hour": 4, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.765, "sample_count": 2220, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 253925, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:53:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.658, "last_frame_pushed_at": "2026-07-06T20:53:24+09:00", "last_frame_read_age_sec": 4.78, "last_frame_read_at": "2026-07-06T20:53:24+09:00", "mjpeg": {"age_sec": 4.639, "exists": true, "mtime": "2026-07-06T20:53:24.907613+09:00", "mtime_epoch": 1783338804.9076128, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:53:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1532.676, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.591, "pushed": 0.591, "read": 0.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.066, "count": 13, "max_ms": 4.506, "p95_ms": 4.506}, "gstreamer_push": {"avg_ms": 1.285, "count": 13, "max_ms": 1.436, "p95_ms": 1.436}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.982, "count": 13, "max_ms": 78.94, "p95_ms": 78.94}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.227, "count": 13, "max_ms": 111.739, "p95_ms": 111.739}, "yolo11_640_fire_11class": {"avg_ms": 68.473, "count": 13, "max_ms": 91.698, "p95_ms": 91.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.197, "count": 13, "max_ms": 92.216, "p95_ms": 92.216}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.117, "count": 13, "max_ms": 125.334, "p95_ms": 125.334}, "read_interval": {"avg_ms": 2475.176, "count": 13, "max_ms": 16164.784, "p95_ms": 16164.784}, "triton_wait": {"avg_ms": 76.565, "count": 13, "max_ms": 111.751, "p95_ms": 111.751}, "visualize": {"avg_ms": 3.588, "count": 13, "max_ms": 3.853, "p95_ms": 3.853}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.006, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5965.965, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x557b7546d200] missing picture in access unit with size 18 [h264 @ 0x557b75edb280] no frame! [in#0/rtsp @ 0x557b75468e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b75468e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557b7546d840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:56:23+09:00", "last_error_summary": "[NULL @ 0x557b7546d200] missing picture in access unit with size 18 [h264 @ 0x557b75edb280] no frame! [in#0/rtsp @ 0x557b75468e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b75468e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557b7546d840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:53:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 175.94, "last_started_at": "2026-07-06T20:53:27+09:00", "pid": 27412, "process": {"exists": false, "pid": 27412}, "restart_count": 50, "restart_count_per_hour": 4, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.307, "sample_count": 2154, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255188, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:56:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T20:56:23+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T20:56:23+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-06T20:56:23.430801+09:00", "mtime_epoch": 1783338983.4308012, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102502}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:56:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1533.449, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.677, "pushed": 0.677, "read": 0.733}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.382, "count": 12, "max_ms": 6.249, "p95_ms": 6.249}, "gstreamer_push": {"avg_ms": 1.302, "count": 12, "max_ms": 1.484, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.942, "count": 12, "max_ms": 79.99, "p95_ms": 79.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.694, "count": 12, "max_ms": 95.928, "p95_ms": 95.928}, "yolo11_640_fire_11class": {"avg_ms": 65.459, "count": 12, "max_ms": 86.258, "p95_ms": 86.258}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.644, "count": 12, "max_ms": 92.954, "p95_ms": 92.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.839, "count": 12, "max_ms": 108.924, "p95_ms": 108.924}, "read_interval": {"avg_ms": 2573.674, "count": 13, "max_ms": 16578.053, "p95_ms": 16578.053}, "triton_wait": {"avg_ms": 74.718, "count": 12, "max_ms": 95.941, "p95_ms": 95.941}, "visualize": {"avg_ms": 3.684, "count": 12, "max_ms": 4.644, "p95_ms": 4.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.734, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5965.367, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T20:56:23+09:00", "last_error_summary": "[NULL @ 0x557b7546d200] missing picture in access unit with size 18 [h264 @ 0x557b75edb280] no frame! [in#0/rtsp @ 0x557b75468e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b75468e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557b7546d840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:56:23+09:00", "last_error_summary": "[NULL @ 0x557b7546d200] missing picture in access unit with size 18 [h264 @ 0x557b75edb280] no frame! [in#0/rtsp @ 0x557b75468e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b75468e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557b7546d840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:56:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 175.979, "last_started_at": "2026-07-06T20:53:27+09:00", "pid": 27412, "process": {"exists": false, "pid": 27412}, "restart_count": 50, "restart_count_per_hour": 4, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.307, "sample_count": 2155, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:56:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T20:56:23+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-06T20:56:23+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-06T20:56:23.430801+09:00", "mtime_epoch": 1783338983.4308012, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102502}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:56:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1533.449, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.325, "count": 13, "max_ms": 6.249, "p95_ms": 6.249}, "gstreamer_push": {"avg_ms": 1.299, "count": 13, "max_ms": 1.484, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.324, "count": 13, "max_ms": 79.99, "p95_ms": 79.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.169, "count": 13, "max_ms": 95.928, "p95_ms": 95.928}, "yolo11_640_fire_11class": {"avg_ms": 66.07, "count": 13, "max_ms": 86.258, "p95_ms": 86.258}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.454, "count": 13, "max_ms": 92.954, "p95_ms": 92.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.16, "count": 13, "max_ms": 108.924, "p95_ms": 108.924}, "read_interval": {"avg_ms": 2573.674, "count": 13, "max_ms": 16578.053, "p95_ms": 16578.053}, "triton_wait": {"avg_ms": 75.115, "count": 13, "max_ms": 95.941, "p95_ms": 95.941}, "visualize": {"avg_ms": 3.666, "count": 13, "max_ms": 4.644, "p95_ms": 4.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.773, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5965.23, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:56:23+09:00", "last_error_summary": "[NULL @ 0x557b7546d200] missing picture in access unit with size 18 [h264 @ 0x557b75edb280] no frame! [in#0/rtsp @ 0x557b75468e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b75468e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557b7546d840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:56:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 175.986, "last_started_at": "2026-07-06T20:53:27+09:00", "pid": 27412, "process": {"exists": false, "pid": 27412}, "restart_count": 50, "restart_count_per_hour": 4, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.307, "sample_count": 2155, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:56:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T20:56:23+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-06T20:56:23+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-06T20:56:23.430801+09:00", "mtime_epoch": 1783338983.4308012, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102502}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:56:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1533.449, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.325, "count": 13, "max_ms": 6.249, "p95_ms": 6.249}, "gstreamer_push": {"avg_ms": 1.299, "count": 13, "max_ms": 1.484, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.324, "count": 13, "max_ms": 79.99, "p95_ms": 79.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.169, "count": 13, "max_ms": 95.928, "p95_ms": 95.928}, "yolo11_640_fire_11class": {"avg_ms": 66.07, "count": 13, "max_ms": 86.258, "p95_ms": 86.258}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.454, "count": 13, "max_ms": 92.954, "p95_ms": 92.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.16, "count": 13, "max_ms": 108.924, "p95_ms": 108.924}, "read_interval": {"avg_ms": 2573.674, "count": 13, "max_ms": 16578.053, "p95_ms": 16578.053}, "triton_wait": {"avg_ms": 75.115, "count": 13, "max_ms": 95.941, "p95_ms": 95.941}, "visualize": {"avg_ms": 3.666, "count": 13, "max_ms": 4.644, "p95_ms": 4.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.78, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5986.484, "limit_mb": 8796093022207.996}, "detail": {"pid": 27564, "restart_count": 51, "start_count": 52}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:56:23+09:00", "last_error_summary": "[NULL @ 0x557b7546d200] missing picture in access unit with size 18 [h264 @ 0x557b75edb280] no frame! [in#0/rtsp @ 0x557b75468e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b75468e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557b7546d840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:56:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:56:25+09:00", "pid": 27564, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.707, "name": "ffmpeg", "pid": 27564, "rss_mb": 57.707, "state": "S (sleeping)", "threads": 1}, "restart_count": 51, "restart_count_per_hour": 5, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.307, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:56:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.186, "last_frame_pushed_at": "2026-07-06T20:56:23+09:00", "last_frame_read_age_sec": 4.28, "last_frame_read_at": "2026-07-06T20:56:23+09:00", "mjpeg": {"age_sec": 4.17, "exists": true, "mtime": "2026-07-06T20:56:23.534803+09:00", "mtime_epoch": 1783338983.534803, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103852}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:56:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1533.449, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.592, "pushed": 0.592, "read": 0.592}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.325, "count": 13, "max_ms": 6.249, "p95_ms": 6.249}, "gstreamer_push": {"avg_ms": 1.299, "count": 13, "max_ms": 1.484, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.324, "count": 13, "max_ms": 79.99, "p95_ms": 79.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.169, "count": 13, "max_ms": 95.928, "p95_ms": 95.928}, "yolo11_640_fire_11class": {"avg_ms": 66.07, "count": 13, "max_ms": 86.258, "p95_ms": 86.258}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.454, "count": 13, "max_ms": 92.954, "p95_ms": 92.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.16, "count": 13, "max_ms": 108.924, "p95_ms": 108.924}, "read_interval": {"avg_ms": 2573.674, "count": 13, "max_ms": 16578.053, "p95_ms": 16578.053}, "triton_wait": {"avg_ms": 75.115, "count": 13, "max_ms": 95.941, "p95_ms": 95.941}, "visualize": {"avg_ms": 3.666, "count": 13, "max_ms": 4.644, "p95_ms": 4.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.953, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6265.57, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:56:23+09:00", "last_error_summary": "[NULL @ 0x557b7546d200] missing picture in access unit with size 18 [h264 @ 0x557b75edb280] no frame! [in#0/rtsp @ 0x557b75468e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b75468e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557b7546d840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:56:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 147.657, "last_started_at": "2026-07-06T20:56:25+09:00", "pid": 27564, "process": {"exists": true, "fd_count": 5, "hwm_mb": 328.188, "name": "ffmpeg", "pid": 27564, "rss_mb": 328.188, "state": "S (sleeping)", "threads": 106}, "restart_count": 51, "restart_count_per_hour": 5, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.5, "sample_count": 2082, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:58:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.059, "last_frame_pushed_at": "2026-07-06T20:58:23+09:00", "last_frame_read_age_sec": 30.159, "last_frame_read_at": "2026-07-06T20:58:23+09:00", "mjpeg": {"age_sec": 30.039, "exists": true, "mtime": "2026-07-06T20:58:23.321364+09:00", "mtime_epoch": 1783339103.3213644, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107381}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:58:53+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1533.871, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6241.582, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:56:23+09:00", "last_error_summary": "[NULL @ 0x557b7546d200] missing picture in access unit with size 18 [h264 @ 0x557b75edb280] no frame! [in#0/rtsp @ 0x557b75468e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b75468e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557b7546d840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T20:56:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 147.669, "last_started_at": "2026-07-06T20:56:25+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 51, "restart_count_per_hour": 5, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.5, "sample_count": 2082, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:58:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.071, "last_frame_pushed_at": "2026-07-06T20:58:23+09:00", "last_frame_read_age_sec": 30.171, "last_frame_read_at": "2026-07-06T20:58:23+09:00", "mjpeg": {"age_sec": 30.051, "exists": true, "mtime": "2026-07-06T20:58:23.321364+09:00", "mtime_epoch": 1783339103.3213644, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107381}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:58:53+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1533.871, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5967.688, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556ca1c5ec40] error while decoding MB 66 21, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:58:53+09:00", "last_error_summary": "[h264 @ 0x556ca1c5ec40] error while decoding MB 66 21, bytestream -5", "last_exit_at": "2026-07-06T20:56:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 147.714, "last_started_at": "2026-07-06T20:56:25+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 51, "restart_count_per_hour": 5, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.5, "sample_count": 2082, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:58:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.115, "last_frame_pushed_at": "2026-07-06T20:58:23+09:00", "last_frame_read_age_sec": 30.216, "last_frame_read_at": "2026-07-06T20:58:23+09:00", "mjpeg": {"age_sec": 30.096, "exists": true, "mtime": "2026-07-06T20:58:23.321364+09:00", "mtime_epoch": 1783339103.3213644, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107381}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:58:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1533.871, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6146.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 27716, "restart_count": 52, "start_count": 53}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T20:58:53+09:00", "last_error_summary": "[h264 @ 0x556ca1c5ec40] error while decoding MB 66 21, bytestream -5", "last_exit_at": "2026-07-06T20:56:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T20:58:56+09:00", "pid": 27716, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.918, "name": "ffmpeg", "pid": 27716, "rss_mb": 206.918, "state": "S (sleeping)", "threads": 106}, "restart_count": 52, "restart_count_per_hour": 6, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.423, "sample_count": 2038, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T20:58:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 35.027, "last_frame_pushed_at": "2026-07-06T20:58:23+09:00", "last_frame_read_age_sec": 35.128, "last_frame_read_at": "2026-07-06T20:58:23+09:00", "mjpeg": {"age_sec": 35.008, "exists": true, "mtime": "2026-07-06T20:58:23.321364+09:00", "mtime_epoch": 1783339103.3213644, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107381}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T20:58:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1533.871, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1903}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6046.746, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5652d4173540] error while decoding MB 44 19, bytestream -8 [h264 @ 0x5652d4d0f140] error while decoding MB 42 42, bytestream -24 [h264 @ 0x5652d411bec0] error while decoding MB 44 3, bytestream -30 [h264 @ 0x5652d415c280] error while decoding MB 77 4, bytestream -14 [h264 @ 0x5652d4234640] error while decoding MB 37 28, bytestream -7 [in#0/rtsp @ 0x5652d40ebe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5652d40ebe00] Error retrieving a packet from demuxer: Connection t", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 360, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:26+09:00", "last_error_summary": "[h264 @ 0x5652d4173540] error while decoding MB 44 19, bytestream -8 [h264 @ 0x5652d4d0f140] error while decoding MB 42 42, bytestream -24 [h264 @ 0x5652d411bec0] error while decoding MB 44 3, bytestream -30 [h264 @ 0x5652d415c280] error while decoding MB 77 4, bytestream -14 [h264 @ 0x5652d4234640] error while decoding MB 37 28, bytestream -7 [in#0/rtsp @ 0x5652d40ebe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5652d40ebe00] Error retrieving a packet from demuxer: Connection t", "last_exit_at": "2026-07-06T20:52:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 526.123, "last_started_at": "2026-07-06T20:52:40+09:00", "pid": 27305, "process": {"exists": false, "pid": 27305}, "restart_count": 120, "restart_count_per_hour": 14, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 121}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.122, "sample_count": 2662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228437, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:01:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T21:01:26+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T21:01:26+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-06T21:01:26.829726+09:00", "mtime_epoch": 1783339286.8297265, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:01:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1646.828, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.272, "pushed": 4.272, "read": 4.272}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.481, "count": 128, "max_ms": 4.942, "p95_ms": 3.809}, "gstreamer_push": {"avg_ms": 1.275, "count": 128, "max_ms": 1.663, "p95_ms": 1.378}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.946, "count": 128, "max_ms": 78.636, "p95_ms": 60.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.031, "count": 128, "max_ms": 85.881, "p95_ms": 80.531}, "yolo11_640_fire_11class": {"avg_ms": 55.2, "count": 128, "max_ms": 83.146, "p95_ms": 61.725}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.097, "count": 128, "max_ms": 84.546, "p95_ms": 67.056}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.693, "count": 128, "max_ms": 97.624, "p95_ms": 92.851}, "read_interval": {"avg_ms": 234.176, "count": 128, "max_ms": 10122.9, "p95_ms": 108.107}, "triton_wait": {"avg_ms": 58.698, "count": 128, "max_ms": 85.892, "p95_ms": 80.749}, "visualize": {"avg_ms": 3.63, "count": 128, "max_ms": 4.684, "p95_ms": 4.14}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1903}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6046.152, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:01:26+09:00", "last_error_summary": "[h264 @ 0x5652d4173540] error while decoding MB 44 19, bytestream -8 [h264 @ 0x5652d4d0f140] error while decoding MB 42 42, bytestream -24 [h264 @ 0x5652d411bec0] error while decoding MB 44 3, bytestream -30 [h264 @ 0x5652d415c280] error while decoding MB 77 4, bytestream -14 [h264 @ 0x5652d4234640] error while decoding MB 37 28, bytestream -7 [in#0/rtsp @ 0x5652d40ebe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5652d40ebe00] Error retrieving a packet from demuxer: Connection t", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 360, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:26+09:00", "last_error_summary": "[h264 @ 0x5652d4173540] error while decoding MB 44 19, bytestream -8 [h264 @ 0x5652d4d0f140] error while decoding MB 42 42, bytestream -24 [h264 @ 0x5652d411bec0] error while decoding MB 44 3, bytestream -30 [h264 @ 0x5652d415c280] error while decoding MB 77 4, bytestream -14 [h264 @ 0x5652d4234640] error while decoding MB 37 28, bytestream -7 [in#0/rtsp @ 0x5652d40ebe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5652d40ebe00] Error retrieving a packet from demuxer: Connection t", "last_exit_at": "2026-07-06T21:01:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 526.183, "last_started_at": "2026-07-06T20:52:40+09:00", "pid": 27305, "process": {"exists": false, "pid": 27305}, "restart_count": 120, "restart_count_per_hour": 14, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 121}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.122, "sample_count": 2662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228438, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:01:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T21:01:26+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T21:01:26+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-06T21:01:26.829726+09:00", "mtime_epoch": 1783339286.8297265, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:01:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1646.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.275, "pushed": 4.275, "read": 4.242}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.478, "count": 128, "max_ms": 4.942, "p95_ms": 3.809}, "gstreamer_push": {"avg_ms": 1.275, "count": 128, "max_ms": 1.663, "p95_ms": 1.378}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.208, "count": 128, "max_ms": 81.773, "p95_ms": 61.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.246, "count": 128, "max_ms": 85.881, "p95_ms": 80.604}, "yolo11_640_fire_11class": {"avg_ms": 55.4, "count": 128, "max_ms": 83.146, "p95_ms": 66.038}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.379, "count": 128, "max_ms": 92.252, "p95_ms": 67.199}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.966, "count": 128, "max_ms": 103.3, "p95_ms": 95.37}, "read_interval": {"avg_ms": 235.381, "count": 127, "max_ms": 10122.9, "p95_ms": 108.107}, "triton_wait": {"avg_ms": 58.976, "count": 128, "max_ms": 92.264, "p95_ms": 82.974}, "visualize": {"avg_ms": 3.628, "count": 128, "max_ms": 4.684, "p95_ms": 4.14}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1903}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6046.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 360, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:26+09:00", "last_error_summary": "[h264 @ 0x5652d4173540] error while decoding MB 44 19, bytestream -8 [h264 @ 0x5652d4d0f140] error while decoding MB 42 42, bytestream -24 [h264 @ 0x5652d411bec0] error while decoding MB 44 3, bytestream -30 [h264 @ 0x5652d415c280] error while decoding MB 77 4, bytestream -14 [h264 @ 0x5652d4234640] error while decoding MB 37 28, bytestream -7 [in#0/rtsp @ 0x5652d40ebe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5652d40ebe00] Error retrieving a packet from demuxer: Connection t", "last_exit_at": "2026-07-06T21:01:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 526.19, "last_started_at": "2026-07-06T20:52:40+09:00", "pid": 27305, "process": {"exists": false, "pid": 27305}, "restart_count": 120, "restart_count_per_hour": 14, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 121}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.122, "sample_count": 2662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228438, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:01:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T21:01:26+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-06T21:01:26+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T21:01:26.829726+09:00", "mtime_epoch": 1783339286.8297265, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:01:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1646.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.274, "pushed": 4.274, "read": 4.241}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.478, "count": 128, "max_ms": 4.942, "p95_ms": 3.809}, "gstreamer_push": {"avg_ms": 1.275, "count": 128, "max_ms": 1.663, "p95_ms": 1.378}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.208, "count": 128, "max_ms": 81.773, "p95_ms": 61.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.246, "count": 128, "max_ms": 85.881, "p95_ms": 80.604}, "yolo11_640_fire_11class": {"avg_ms": 55.4, "count": 128, "max_ms": 83.146, "p95_ms": 66.038}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.379, "count": 128, "max_ms": 92.252, "p95_ms": 67.199}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.966, "count": 128, "max_ms": 103.3, "p95_ms": 95.37}, "read_interval": {"avg_ms": 235.381, "count": 127, "max_ms": 10122.9, "p95_ms": 108.107}, "triton_wait": {"avg_ms": 58.976, "count": 128, "max_ms": 92.264, "p95_ms": 82.974}, "visualize": {"avg_ms": 3.628, "count": 128, "max_ms": 4.684, "p95_ms": 4.14}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5723.055, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f14fbd9a00] error while decoding MB 19 21, bytestream -7 [in#0/rtsp @ 0x55f14f237e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f14f237e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:27+09:00", "last_error_summary": "[h264 @ 0x55f14fbd9a00] error while decoding MB 19 21, bytestream -7 [in#0/rtsp @ 0x55f14f237e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f14f237e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T20:56:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.939, "last_started_at": "2026-07-06T20:58:56+09:00", "pid": 27716, "process": {"exists": false, "pid": 27716}, "restart_count": 52, "restart_count_per_hour": 6, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.244, "sample_count": 2078, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 257267, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:01:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T21:01:27+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T21:01:27+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-06T21:01:27.205735+09:00", "mtime_epoch": 1783339287.205735, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95146}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:01:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1534.367, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.686, "pushed": 0.686, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.974, "count": 12, "max_ms": 6.348, "p95_ms": 6.348}, "gstreamer_push": {"avg_ms": 1.109, "count": 12, "max_ms": 1.377, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.044, "count": 12, "max_ms": 78.86, "p95_ms": 78.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.165, "count": 12, "max_ms": 87.932, "p95_ms": 87.932}, "yolo11_640_fire_11class": {"avg_ms": 69.431, "count": 12, "max_ms": 85.143, "p95_ms": 85.143}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.713, "count": 12, "max_ms": 95.908, "p95_ms": 95.908}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.297, "count": 12, "max_ms": 108.623, "p95_ms": 108.623}, "read_interval": {"avg_ms": 2627.726, "count": 13, "max_ms": 16743.448, "p95_ms": 16743.448}, "triton_wait": {"avg_ms": 76.821, "count": 12, "max_ms": 95.921, "p95_ms": 95.921}, "visualize": {"avg_ms": 3.098, "count": 12, "max_ms": 4.29, "p95_ms": 4.29}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.482, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5722.98, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:01:27+09:00", "last_error_summary": "[h264 @ 0x55f14fbd9a00] error while decoding MB 19 21, bytestream -7 [in#0/rtsp @ 0x55f14f237e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f14f237e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:27+09:00", "last_error_summary": "[h264 @ 0x55f14fbd9a00] error while decoding MB 19 21, bytestream -7 [in#0/rtsp @ 0x55f14f237e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f14f237e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:01:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.961, "last_started_at": "2026-07-06T20:58:56+09:00", "pid": 27716, "process": {"exists": false, "pid": 27716}, "restart_count": 52, "restart_count_per_hour": 6, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.244, "sample_count": 2079, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 257268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:01:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T21:01:27+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-06T21:01:27+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-06T21:01:27.205735+09:00", "mtime_epoch": 1783339287.205735, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95146}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:01:27+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1534.367, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.743, "pushed": 0.743, "read": 0.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.361, "count": 13, "max_ms": 9.007, "p95_ms": 9.007}, "gstreamer_push": {"avg_ms": 1.105, "count": 13, "max_ms": 1.377, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.544, "count": 13, "max_ms": 78.86, "p95_ms": 78.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.998, "count": 13, "max_ms": 87.932, "p95_ms": 87.932}, "yolo11_640_fire_11class": {"avg_ms": 68.506, "count": 13, "max_ms": 85.143, "p95_ms": 85.143}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.577, "count": 13, "max_ms": 95.908, "p95_ms": 95.908}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.579, "count": 13, "max_ms": 108.623, "p95_ms": 108.623}, "read_interval": {"avg_ms": 2627.726, "count": 13, "max_ms": 16743.448, "p95_ms": 16743.448}, "triton_wait": {"avg_ms": 75.528, "count": 13, "max_ms": 95.921, "p95_ms": 95.921}, "visualize": {"avg_ms": 3.281, "count": 13, "max_ms": 5.474, "p95_ms": 5.474}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.503, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5722.828, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:27+09:00", "last_error_summary": "[h264 @ 0x55f14fbd9a00] error while decoding MB 19 21, bytestream -7 [in#0/rtsp @ 0x55f14f237e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f14f237e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:01:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.968, "last_started_at": "2026-07-06T20:58:56+09:00", "pid": 27716, "process": {"exists": false, "pid": 27716}, "restart_count": 52, "restart_count_per_hour": 6, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.244, "sample_count": 2079, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 257268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:01:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T21:01:27+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-06T21:01:27+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-06T21:01:27.205735+09:00", "mtime_epoch": 1783339287.205735, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95146}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:01:27+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1534.367, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.742, "pushed": 0.742, "read": 0.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.361, "count": 13, "max_ms": 9.007, "p95_ms": 9.007}, "gstreamer_push": {"avg_ms": 1.105, "count": 13, "max_ms": 1.377, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.544, "count": 13, "max_ms": 78.86, "p95_ms": 78.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.998, "count": 13, "max_ms": 87.932, "p95_ms": 87.932}, "yolo11_640_fire_11class": {"avg_ms": 68.506, "count": 13, "max_ms": 85.143, "p95_ms": 85.143}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.577, "count": 13, "max_ms": 95.908, "p95_ms": 95.908}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.579, "count": 13, "max_ms": 108.623, "p95_ms": 108.623}, "read_interval": {"avg_ms": 2627.726, "count": 13, "max_ms": 16743.448, "p95_ms": 16743.448}, "triton_wait": {"avg_ms": 75.528, "count": 13, "max_ms": 95.921, "p95_ms": 95.921}, "visualize": {"avg_ms": 3.281, "count": 13, "max_ms": 5.474, "p95_ms": 5.474}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.51, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1903}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5853.441, "limit_mb": 8796093022207.996}, "detail": {"pid": 27868, "restart_count": 121, "start_count": 122}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 360, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:26+09:00", "last_error_summary": "[h264 @ 0x5652d4173540] error while decoding MB 44 19, bytestream -8 [h264 @ 0x5652d4d0f140] error while decoding MB 42 42, bytestream -24 [h264 @ 0x5652d411bec0] error while decoding MB 44 3, bytestream -30 [h264 @ 0x5652d415c280] error while decoding MB 77 4, bytestream -14 [h264 @ 0x5652d4234640] error while decoding MB 37 28, bytestream -7 [in#0/rtsp @ 0x5652d40ebe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5652d40ebe00] Error retrieving a packet from demuxer: Connection t", "last_exit_at": "2026-07-06T21:01:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:01:29+09:00", "pid": 27868, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.125, "name": "ffmpeg", "pid": 27868, "rss_mb": 139.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 121, "restart_count_per_hour": 15, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 122}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.054, "sample_count": 2619, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228438, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:01:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.304, "last_frame_pushed_at": "2026-07-06T21:01:26+09:00", "last_frame_read_age_sec": 4.409, "last_frame_read_at": "2026-07-06T21:01:26+09:00", "mjpeg": {"age_sec": 4.285, "exists": true, "mtime": "2026-07-06T21:01:26.941729+09:00", "mtime_epoch": 1783339286.941729, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126458}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:01:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1646.828, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 75, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.539, "pushed": 2.539, "read": 2.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.475, "count": 76, "max_ms": 4.942, "p95_ms": 4.102}, "gstreamer_push": {"avg_ms": 1.281, "count": 76, "max_ms": 1.663, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.454, "count": 76, "max_ms": 81.773, "p95_ms": 67.826}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.839, "count": 76, "max_ms": 85.192, "p95_ms": 80.735}, "yolo11_640_fire_11class": {"avg_ms": 56.183, "count": 76, "max_ms": 83.146, "p95_ms": 74.805}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.341, "count": 76, "max_ms": 92.252, "p95_ms": 82.477}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.876, "count": 76, "max_ms": 103.3, "p95_ms": 96.217}, "read_interval": {"avg_ms": 341.259, "count": 75, "max_ms": 10122.9, "p95_ms": 109.999}, "triton_wait": {"avg_ms": 59.856, "count": 76, "max_ms": 92.264, "p95_ms": 84.498}, "visualize": {"avg_ms": 3.648, "count": 76, "max_ms": 4.395, "p95_ms": 4.213}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5994.742, "limit_mb": 8796093022207.996}, "detail": {"pid": 27870, "restart_count": 53, "start_count": 54}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:27+09:00", "last_error_summary": "[h264 @ 0x55f14fbd9a00] error while decoding MB 19 21, bytestream -7 [in#0/rtsp @ 0x55f14f237e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f14f237e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:01:30+09:00", "pid": 27870, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.184, "name": "ffmpeg", "pid": 27870, "rss_mb": 187.184, "state": "S (sleeping)", "threads": 106}, "restart_count": 53, "restart_count_per_hour": 7, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.998, "sample_count": 2043, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 257268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:01:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.906, "last_frame_pushed_at": "2026-07-06T21:01:27+09:00", "last_frame_read_age_sec": 4.986, "last_frame_read_at": "2026-07-06T21:01:27+09:00", "mjpeg": {"age_sec": 4.892, "exists": true, "mtime": "2026-07-06T21:01:27.297737+09:00", "mtime_epoch": 1783339287.2977371, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96066}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:01:32+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1534.367, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.58, "pushed": 0.58, "read": 0.58}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.361, "count": 13, "max_ms": 9.007, "p95_ms": 9.007}, "gstreamer_push": {"avg_ms": 1.105, "count": 13, "max_ms": 1.377, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.544, "count": 13, "max_ms": 78.86, "p95_ms": 78.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.998, "count": 13, "max_ms": 87.932, "p95_ms": 87.932}, "yolo11_640_fire_11class": {"avg_ms": 68.506, "count": 13, "max_ms": 85.143, "p95_ms": 85.143}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.577, "count": 13, "max_ms": 95.908, "p95_ms": 95.908}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.579, "count": 13, "max_ms": 108.623, "p95_ms": 108.623}, "read_interval": {"avg_ms": 2627.726, "count": 13, "max_ms": 16743.448, "p95_ms": 16743.448}, "triton_wait": {"avg_ms": 75.528, "count": 13, "max_ms": 95.921, "p95_ms": 95.921}, "visualize": {"avg_ms": 3.281, "count": 13, "max_ms": 5.474, "p95_ms": 5.474}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.404, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6269.992, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:27+09:00", "last_error_summary": "[h264 @ 0x55f14fbd9a00] error while decoding MB 19 21, bytestream -7 [in#0/rtsp @ 0x55f14f237e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f14f237e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.127, "last_started_at": "2026-07-06T21:01:30+09:00", "pid": 27870, "process": {"exists": true, "fd_count": 5, "hwm_mb": 330.797, "name": "ffmpeg", "pid": 27870, "rss_mb": 330.797, "state": "S (sleeping)", "threads": 106}, "restart_count": 53, "restart_count_per_hour": 7, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.711, "sample_count": 1949, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 258166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:03:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.059, "last_frame_pushed_at": "2026-07-06T21:03:12+09:00", "last_frame_read_age_sec": 30.175, "last_frame_read_at": "2026-07-06T21:03:12+09:00", "mjpeg": {"age_sec": 30.048, "exists": true, "mtime": "2026-07-06T21:03:12.268100+09:00", "mtime_epoch": 1783339392.2680995, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:03:42+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1534.723, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6260.234, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:01:27+09:00", "last_error_summary": "[h264 @ 0x55f14fbd9a00] error while decoding MB 19 21, bytestream -7 [in#0/rtsp @ 0x55f14f237e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f14f237e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.139, "last_started_at": "2026-07-06T21:01:30+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 53, "restart_count_per_hour": 7, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.711, "sample_count": 1949, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 258166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:03:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.071, "last_frame_pushed_at": "2026-07-06T21:03:12+09:00", "last_frame_read_age_sec": 30.187, "last_frame_read_at": "2026-07-06T21:03:12+09:00", "mjpeg": {"age_sec": 30.06, "exists": true, "mtime": "2026-07-06T21:03:12.268100+09:00", "mtime_epoch": 1783339392.2680995, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:03:42+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1534.723, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5969.035, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559ce1dc9900] error while decoding MB 86 29, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:03:42+09:00", "last_error_summary": "[h264 @ 0x559ce1dc9900] error while decoding MB 86 29, bytestream -7", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.189, "last_started_at": "2026-07-06T21:01:30+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 53, "restart_count_per_hour": 7, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.711, "sample_count": 1949, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 258166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:03:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.121, "last_frame_pushed_at": "2026-07-06T21:03:12+09:00", "last_frame_read_age_sec": 30.237, "last_frame_read_at": "2026-07-06T21:03:12+09:00", "mjpeg": {"age_sec": 30.109, "exists": true, "mtime": "2026-07-06T21:03:12.268100+09:00", "mtime_epoch": 1783339392.2680995, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:03:42+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1534.723, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6052.293, "limit_mb": 8796093022207.996}, "detail": {"pid": 28127, "restart_count": 54, "start_count": 55}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:03:42+09:00", "last_error_summary": "[h264 @ 0x559ce1dc9900] error while decoding MB 86 29, bytestream -7", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:03:44+09:00", "pid": 28127, "process": {"exists": true, "fd_count": 5, "hwm_mb": 121.57, "name": "ffmpeg", "pid": 28127, "rss_mb": 121.57, "state": "S (sleeping)", "threads": 19}, "restart_count": 54, "restart_count_per_hour": 8, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.711, "sample_count": 1949, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 258166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:03:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.702, "last_frame_pushed_at": "2026-07-06T21:03:12+09:00", "last_frame_read_age_sec": 34.819, "last_frame_read_at": "2026-07-06T21:03:12+09:00", "mjpeg": {"age_sec": 34.691, "exists": true, "mtime": "2026-07-06T21:03:12.268100+09:00", "mtime_epoch": 1783339392.2680995, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:03:46+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1534.723, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2169}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6275.539, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:03:42+09:00", "last_error_summary": "[h264 @ 0x559ce1dc9900] error while decoding MB 86 29, bytestream -7", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 271.608, "last_started_at": "2026-07-06T21:03:44+09:00", "pid": 28127, "process": {"exists": true, "fd_count": 5, "hwm_mb": 330.301, "name": "ffmpeg", "pid": 28127, "rss_mb": 330.301, "state": "S (sleeping)", "threads": 106}, "restart_count": 54, "restart_count_per_hour": 8, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.642, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 260304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:07:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.059, "last_frame_pushed_at": "2026-07-06T21:07:46+09:00", "last_frame_read_age_sec": 30.15, "last_frame_read_at": "2026-07-06T21:07:46+09:00", "mjpeg": {"age_sec": 30.045, "exists": true, "mtime": "2026-07-06T21:07:46.521955+09:00", "mtime_epoch": 1783339666.5219553, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:08:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1535.75, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2169}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6263.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:03:42+09:00", "last_error_summary": "[h264 @ 0x559ce1dc9900] error while decoding MB 86 29, bytestream -7", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 271.621, "last_started_at": "2026-07-06T21:03:44+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 54, "restart_count_per_hour": 8, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.642, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 260304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:07:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.071, "last_frame_pushed_at": "2026-07-06T21:07:46+09:00", "last_frame_read_age_sec": 30.163, "last_frame_read_at": "2026-07-06T21:07:46+09:00", "mjpeg": {"age_sec": 30.057, "exists": true, "mtime": "2026-07-06T21:07:46.521955+09:00", "mtime_epoch": 1783339666.5219553, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:08:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1535.75, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2169}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5974.285, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562fd8966f40] error while decoding MB 48 36, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:08:16+09:00", "last_error_summary": "[h264 @ 0x562fd8966f40] error while decoding MB 48 36, bytestream -7", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 271.671, "last_started_at": "2026-07-06T21:03:44+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 54, "restart_count_per_hour": 8, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.642, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 260304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:07:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.122, "last_frame_pushed_at": "2026-07-06T21:07:46+09:00", "last_frame_read_age_sec": 30.214, "last_frame_read_at": "2026-07-06T21:07:46+09:00", "mjpeg": {"age_sec": 30.108, "exists": true, "mtime": "2026-07-06T21:07:46.521955+09:00", "mtime_epoch": 1783339666.5219553, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:08:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1535.75, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2169}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5996.859, "limit_mb": 8796093022207.996}, "detail": {"pid": 28279, "restart_count": 55, "start_count": 56}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:08:16+09:00", "last_error_summary": "[h264 @ 0x562fd8966f40] error while decoding MB 48 36, bytestream -7", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:08:18+09:00", "pid": 28279, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.914, "name": "ffmpeg", "pid": 28279, "rss_mb": 57.914, "state": "S (sleeping)", "threads": 1}, "restart_count": 55, "restart_count_per_hour": 9, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.642, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 260304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:07:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.308, "last_frame_pushed_at": "2026-07-06T21:07:46+09:00", "last_frame_read_age_sec": 34.4, "last_frame_read_at": "2026-07-06T21:07:46+09:00", "mjpeg": {"age_sec": 34.294, "exists": true, "mtime": "2026-07-06T21:07:46.521955+09:00", "mtime_epoch": 1783339666.5219553, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:08:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1535.75, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1946}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6032.289, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561a7871a8c0] reference picture missing during reorder [h264 @ 0x561a7871a8c0] Missing reference picture, default is 65569 [h264 @ 0x561a7876ebc0] mmco: unref short failure [h264 @ 0x561a7876ebc0] illegal short term buffer state detected [h264 @ 0x561a79241f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561a79241f00] error while decoding MB 0 38, bytestream 2412 [h264 @ 0x561a7876ebc0] cabac decode of qscale diff failed at 65 39 [h264 @ 0x561a7876ebc0] error while ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 112, "h264_decode_error_count": 365, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:11:03+09:00", "last_error_summary": "[h264 @ 0x561a7871a8c0] reference picture missing during reorder [h264 @ 0x561a7871a8c0] Missing reference picture, default is 65569 [h264 @ 0x561a7876ebc0] mmco: unref short failure [h264 @ 0x561a7876ebc0] illegal short term buffer state detected [h264 @ 0x561a79241f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561a79241f00] error while decoding MB 0 38, bytestream 2412 [h264 @ 0x561a7876ebc0] cabac decode of qscale diff failed at 65 39 [h264 @ 0x561a7876ebc0] error while ", "last_exit_at": "2026-07-06T21:01:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 573.842, "last_started_at": "2026-07-06T21:01:29+09:00", "pid": 27868, "process": {"exists": false, "pid": 27868}, "restart_count": 121, "restart_count_per_hour": 11, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 122}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.99, "sample_count": 2599, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 233587, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:11:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T21:11:03+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T21:11:03+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-06T21:11:03.021956+09:00", "mtime_epoch": 1783339863.0219562, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129030}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:11:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1649.406, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.277, "pushed": 3.277, "read": 3.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.784, "count": 98, "max_ms": 5.893, "p95_ms": 5.724}, "gstreamer_push": {"avg_ms": 1.406, "count": 98, "max_ms": 2.478, "p95_ms": 2.194}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.592, "count": 98, "max_ms": 93.945, "p95_ms": 84.247}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.211, "count": 98, "max_ms": 114.56, "p95_ms": 92.533}, "yolo11_640_fire_11class": {"avg_ms": 74.885, "count": 98, "max_ms": 95.044, "p95_ms": 87.738}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.81, "count": 98, "max_ms": 106.297, "p95_ms": 100.841}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.746, "count": 98, "max_ms": 127.278, "p95_ms": 114.087}, "read_interval": {"avg_ms": 305.606, "count": 98, "max_ms": 10126.837, "p95_ms": 128.709}, "triton_wait": {"avg_ms": 84.818, "count": 98, "max_ms": 114.573, "p95_ms": 101.718}, "visualize": {"avg_ms": 3.869, "count": 98, "max_ms": 6.425, "p95_ms": 6.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1946}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6031.805, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:11:03+09:00", "last_error_summary": "[h264 @ 0x561a7871a8c0] reference picture missing during reorder [h264 @ 0x561a7871a8c0] Missing reference picture, default is 65569 [h264 @ 0x561a7876ebc0] mmco: unref short failure [h264 @ 0x561a7876ebc0] illegal short term buffer state detected [h264 @ 0x561a79241f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561a79241f00] error while decoding MB 0 38, bytestream 2412 [h264 @ 0x561a7876ebc0] cabac decode of qscale diff failed at 65 39 [h264 @ 0x561a7876ebc0] error while ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 112, "h264_decode_error_count": 365, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:11:03+09:00", "last_error_summary": "[h264 @ 0x561a7871a8c0] reference picture missing during reorder [h264 @ 0x561a7871a8c0] Missing reference picture, default is 65569 [h264 @ 0x561a7876ebc0] mmco: unref short failure [h264 @ 0x561a7876ebc0] illegal short term buffer state detected [h264 @ 0x561a79241f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561a79241f00] error while decoding MB 0 38, bytestream 2412 [h264 @ 0x561a7876ebc0] cabac decode of qscale diff failed at 65 39 [h264 @ 0x561a7876ebc0] error while ", "last_exit_at": "2026-07-06T21:11:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 573.906, "last_started_at": "2026-07-06T21:01:29+09:00", "pid": 27868, "process": {"exists": false, "pid": 27868}, "restart_count": 121, "restart_count_per_hour": 11, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 122}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.99, "sample_count": 2599, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 233588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:11:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T21:11:03+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T21:11:03+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-06T21:11:03.021956+09:00", "mtime_epoch": 1783339863.0219562, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129030}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:11:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1649.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.303, "pushed": 3.303, "read": 3.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.78, "count": 99, "max_ms": 5.893, "p95_ms": 5.724}, "gstreamer_push": {"avg_ms": 1.404, "count": 99, "max_ms": 2.478, "p95_ms": 2.194}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.753, "count": 99, "max_ms": 93.945, "p95_ms": 84.999}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.405, "count": 99, "max_ms": 114.56, "p95_ms": 94.115}, "yolo11_640_fire_11class": {"avg_ms": 74.993, "count": 99, "max_ms": 95.044, "p95_ms": 87.738}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.927, "count": 99, "max_ms": 106.297, "p95_ms": 100.841}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.858, "count": 99, "max_ms": 127.278, "p95_ms": 114.087}, "read_interval": {"avg_ms": 305.606, "count": 98, "max_ms": 10126.837, "p95_ms": 128.709}, "triton_wait": {"avg_ms": 84.935, "count": 99, "max_ms": 114.573, "p95_ms": 101.718}, "visualize": {"avg_ms": 3.87, "count": 99, "max_ms": 6.425, "p95_ms": 6.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1946}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6031.918, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 112, "h264_decode_error_count": 365, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:11:03+09:00", "last_error_summary": "[h264 @ 0x561a7871a8c0] reference picture missing during reorder [h264 @ 0x561a7871a8c0] Missing reference picture, default is 65569 [h264 @ 0x561a7876ebc0] mmco: unref short failure [h264 @ 0x561a7876ebc0] illegal short term buffer state detected [h264 @ 0x561a79241f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561a79241f00] error while decoding MB 0 38, bytestream 2412 [h264 @ 0x561a7876ebc0] cabac decode of qscale diff failed at 65 39 [h264 @ 0x561a7876ebc0] error while ", "last_exit_at": "2026-07-06T21:11:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 573.914, "last_started_at": "2026-07-06T21:01:29+09:00", "pid": 27868, "process": {"exists": false, "pid": 27868}, "restart_count": 121, "restart_count_per_hour": 11, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 122}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.99, "sample_count": 2599, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 233588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:11:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T21:11:03+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T21:11:03+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T21:11:03.021956+09:00", "mtime_epoch": 1783339863.0219562, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129030}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:11:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1649.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.302, "pushed": 3.302, "read": 3.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.78, "count": 99, "max_ms": 5.893, "p95_ms": 5.724}, "gstreamer_push": {"avg_ms": 1.404, "count": 99, "max_ms": 2.478, "p95_ms": 2.194}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.753, "count": 99, "max_ms": 93.945, "p95_ms": 84.999}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.405, "count": 99, "max_ms": 114.56, "p95_ms": 94.115}, "yolo11_640_fire_11class": {"avg_ms": 74.993, "count": 99, "max_ms": 95.044, "p95_ms": 87.738}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.927, "count": 99, "max_ms": 106.297, "p95_ms": 100.841}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.858, "count": 99, "max_ms": 127.278, "p95_ms": 114.087}, "read_interval": {"avg_ms": 305.606, "count": 98, "max_ms": 10126.837, "p95_ms": 128.709}, "triton_wait": {"avg_ms": 84.935, "count": 99, "max_ms": 114.573, "p95_ms": 101.718}, "visualize": {"avg_ms": 3.87, "count": 99, "max_ms": 6.425, "p95_ms": 6.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1946}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6130.105, "limit_mb": 8796093022207.996}, "detail": {"pid": 28431, "restart_count": 122, "start_count": 123}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 112, "h264_decode_error_count": 365, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:11:03+09:00", "last_error_summary": "[h264 @ 0x561a7871a8c0] reference picture missing during reorder [h264 @ 0x561a7871a8c0] Missing reference picture, default is 65569 [h264 @ 0x561a7876ebc0] mmco: unref short failure [h264 @ 0x561a7876ebc0] illegal short term buffer state detected [h264 @ 0x561a79241f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561a79241f00] error while decoding MB 0 38, bytestream 2412 [h264 @ 0x561a7876ebc0] cabac decode of qscale diff failed at 65 39 [h264 @ 0x561a7876ebc0] error while ", "last_exit_at": "2026-07-06T21:11:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:11:06+09:00", "pid": 28431, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.641, "name": "ffmpeg", "pid": 28431, "rss_mb": 139.641, "state": "S (sleeping)", "threads": 106}, "restart_count": 122, "restart_count_per_hour": 12, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 123}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.051, "sample_count": 2555, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 233588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:11:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.877, "last_frame_pushed_at": "2026-07-06T21:11:03+09:00", "last_frame_read_age_sec": 4.988, "last_frame_read_at": "2026-07-06T21:11:03+09:00", "mjpeg": {"age_sec": 4.861, "exists": true, "mtime": "2026-07-06T21:11:03.141959+09:00", "mtime_epoch": 1783339863.1419587, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:11:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1649.406, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.869, "pushed": 1.869, "read": 1.836}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.877, "count": 56, "max_ms": 5.893, "p95_ms": 5.782}, "gstreamer_push": {"avg_ms": 1.443, "count": 56, "max_ms": 2.478, "p95_ms": 2.214}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.95, "count": 56, "max_ms": 93.945, "p95_ms": 85.53}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.605, "count": 56, "max_ms": 114.56, "p95_ms": 96.374}, "yolo11_640_fire_11class": {"avg_ms": 73.291, "count": 56, "max_ms": 95.044, "p95_ms": 91.7}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.384, "count": 56, "max_ms": 106.297, "p95_ms": 102.608}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.347, "count": 56, "max_ms": 127.278, "p95_ms": 115.901}, "read_interval": {"avg_ms": 455.852, "count": 55, "max_ms": 10126.837, "p95_ms": 139.887}, "triton_wait": {"avg_ms": 83.122, "count": 56, "max_ms": 114.573, "p95_ms": 103.862}, "visualize": {"avg_ms": 3.985, "count": 56, "max_ms": 6.425, "p95_ms": 6.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5972.566, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x56182b28b200] missing picture in access unit with size 18 [h264 @ 0x56182bea2b80] no frame! [in#0/rtsp @ 0x56182b286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56182b286e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56182b28b840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:11:52+09:00", "last_error_summary": "[NULL @ 0x56182b28b200] missing picture in access unit with size 18 [h264 @ 0x56182bea2b80] no frame! [in#0/rtsp @ 0x56182b286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56182b286e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56182b28b840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 213.608, "last_started_at": "2026-07-06T21:08:18+09:00", "pid": 28279, "process": {"exists": false, "pid": 28279}, "restart_count": 55, "restart_count_per_hour": 9, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.287, "sample_count": 2168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261988, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:11:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-06T21:11:52+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-06T21:11:52+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-06T21:11:52.354943+09:00", "mtime_epoch": 1783339912.3549428, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102174}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:11:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1536.523, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.474, "pushed": 0.474, "read": 0.51}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.723, "count": 13, "max_ms": 6.01, "p95_ms": 6.01}, "gstreamer_push": {"avg_ms": 1.303, "count": 13, "max_ms": 1.424, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.049, "count": 13, "max_ms": 86.589, "p95_ms": 86.589}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.41, "count": 13, "max_ms": 88.117, "p95_ms": 88.117}, "yolo11_640_fire_11class": {"avg_ms": 69.859, "count": 13, "max_ms": 86.602, "p95_ms": 86.602}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.173, "count": 13, "max_ms": 95.18, "p95_ms": 95.18}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.983, "count": 13, "max_ms": 108.628, "p95_ms": 108.628}, "read_interval": {"avg_ms": 2374.552, "count": 14, "max_ms": 16076.501, "p95_ms": 16076.501}, "triton_wait": {"avg_ms": 79.434, "count": 13, "max_ms": 95.192, "p95_ms": 95.192}, "visualize": {"avg_ms": 3.727, "count": 13, "max_ms": 4.112, "p95_ms": 4.112}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.431, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5971.762, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:11:52+09:00", "last_error_summary": "[NULL @ 0x56182b28b200] missing picture in access unit with size 18 [h264 @ 0x56182bea2b80] no frame! [in#0/rtsp @ 0x56182b286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56182b286e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56182b28b840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:11:52+09:00", "last_error_summary": "[NULL @ 0x56182b28b200] missing picture in access unit with size 18 [h264 @ 0x56182bea2b80] no frame! [in#0/rtsp @ 0x56182b286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56182b286e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56182b28b840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:11:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 213.636, "last_started_at": "2026-07-06T21:08:18+09:00", "pid": 28279, "process": {"exists": false, "pid": 28279}, "restart_count": 55, "restart_count_per_hour": 9, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.287, "sample_count": 2168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261989, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:11:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T21:11:52+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-06T21:11:52+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-06T21:11:52.354943+09:00", "mtime_epoch": 1783339912.3549428, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102174}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:11:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1536.523, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.51, "pushed": 0.51, "read": 0.51}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.676, "count": 14, "max_ms": 6.01, "p95_ms": 6.01}, "gstreamer_push": {"avg_ms": 1.306, "count": 14, "max_ms": 1.424, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.206, "count": 14, "max_ms": 86.589, "p95_ms": 86.589}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.059, "count": 14, "max_ms": 88.117, "p95_ms": 88.117}, "yolo11_640_fire_11class": {"avg_ms": 69.832, "count": 14, "max_ms": 86.602, "p95_ms": 86.602}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.386, "count": 14, "max_ms": 95.18, "p95_ms": 95.18}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.92, "count": 14, "max_ms": 108.628, "p95_ms": 108.628}, "read_interval": {"avg_ms": 2374.552, "count": 14, "max_ms": 16076.501, "p95_ms": 16076.501}, "triton_wait": {"avg_ms": 79.415, "count": 14, "max_ms": 95.192, "p95_ms": 95.192}, "visualize": {"avg_ms": 3.72, "count": 14, "max_ms": 4.112, "p95_ms": 4.112}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.459, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5971.652, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:11:52+09:00", "last_error_summary": "[NULL @ 0x56182b28b200] missing picture in access unit with size 18 [h264 @ 0x56182bea2b80] no frame! [in#0/rtsp @ 0x56182b286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56182b286e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56182b28b840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:11:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 213.643, "last_started_at": "2026-07-06T21:08:18+09:00", "pid": 28279, "process": {"exists": false, "pid": 28279}, "restart_count": 55, "restart_count_per_hour": 9, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.287, "sample_count": 2168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261989, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:11:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T21:11:52+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-06T21:11:52+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-06T21:11:52.354943+09:00", "mtime_epoch": 1783339912.3549428, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102174}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:11:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1536.523, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.51, "pushed": 0.51, "read": 0.51}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.676, "count": 14, "max_ms": 6.01, "p95_ms": 6.01}, "gstreamer_push": {"avg_ms": 1.306, "count": 14, "max_ms": 1.424, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.206, "count": 14, "max_ms": 86.589, "p95_ms": 86.589}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.059, "count": 14, "max_ms": 88.117, "p95_ms": 88.117}, "yolo11_640_fire_11class": {"avg_ms": 69.832, "count": 14, "max_ms": 86.602, "p95_ms": 86.602}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.386, "count": 14, "max_ms": 95.18, "p95_ms": 95.18}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.92, "count": 14, "max_ms": 108.628, "p95_ms": 108.628}, "read_interval": {"avg_ms": 2374.552, "count": 14, "max_ms": 16076.501, "p95_ms": 16076.501}, "triton_wait": {"avg_ms": 79.415, "count": 14, "max_ms": 95.192, "p95_ms": 95.192}, "visualize": {"avg_ms": 3.72, "count": 14, "max_ms": 4.112, "p95_ms": 4.112}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.466, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2183}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5989.641, "limit_mb": 8796093022207.996}, "detail": {"pid": 28538, "restart_count": 56, "start_count": 57}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:11:52+09:00", "last_error_summary": "[NULL @ 0x56182b28b200] missing picture in access unit with size 18 [h264 @ 0x56182bea2b80] no frame! [in#0/rtsp @ 0x56182b286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56182b286e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56182b28b840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:11:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:11:54+09:00", "pid": 28538, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.344, "name": "ffmpeg", "pid": 28538, "rss_mb": 57.344, "state": "S (sleeping)", "threads": 1}, "restart_count": 56, "restart_count_per_hour": 10, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.29, "sample_count": 2131, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261989, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:11:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.159, "last_frame_pushed_at": "2026-07-06T21:11:52+09:00", "last_frame_read_age_sec": 4.253, "last_frame_read_at": "2026-07-06T21:11:52+09:00", "mjpeg": {"age_sec": 4.142, "exists": true, "mtime": "2026-07-06T21:11:52.462945+09:00", "mtime_epoch": 1783339912.462945, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102762}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:11:56+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1536.523, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.605, "pushed": 0.605, "read": 0.605}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.716, "count": 13, "max_ms": 6.01, "p95_ms": 6.01}, "gstreamer_push": {"avg_ms": 1.306, "count": 13, "max_ms": 1.424, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.012, "count": 13, "max_ms": 86.589, "p95_ms": 86.589}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.548, "count": 13, "max_ms": 88.117, "p95_ms": 88.117}, "yolo11_640_fire_11class": {"avg_ms": 71.101, "count": 13, "max_ms": 86.602, "p95_ms": 86.602}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.053, "count": 13, "max_ms": 95.18, "p95_ms": 95.18}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.886, "count": 13, "max_ms": 108.628, "p95_ms": 108.628}, "read_interval": {"avg_ms": 2104.564, "count": 13, "max_ms": 16076.501, "p95_ms": 16076.501}, "triton_wait": {"avg_ms": 81.315, "count": 13, "max_ms": 95.192, "p95_ms": 95.192}, "visualize": {"avg_ms": 3.731, "count": 13, "max_ms": 4.112, "p95_ms": 4.112}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.496, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2210}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5976.777, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c552966780] error while decoding MB 95 19, bytestream -5 [in#0/rtsp @ 0x55c551d52e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c551d52e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:18:33+09:00", "last_error_summary": "[h264 @ 0x55c552966780] error while decoding MB 95 19, bytestream -5 [in#0/rtsp @ 0x55c551d52e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c551d52e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:11:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 398.792, "last_started_at": "2026-07-06T21:11:54+09:00", "pid": 28538, "process": {"exists": false, "pid": 28538}, "restart_count": 56, "restart_count_per_hour": 10, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.622, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 265264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:18:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T21:18:33+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T21:18:33+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T21:18:33.338784+09:00", "mtime_epoch": 1783340313.3387837, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:18:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1539.359, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.687, "pushed": 0.687, "read": 0.745}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.142, "count": 12, "max_ms": 4.737, "p95_ms": 4.737}, "gstreamer_push": {"avg_ms": 1.252, "count": 12, "max_ms": 1.356, "p95_ms": 1.356}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.387, "count": 12, "max_ms": 82.768, "p95_ms": 82.768}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.134, "count": 12, "max_ms": 104.805, "p95_ms": 104.805}, "yolo11_640_fire_11class": {"avg_ms": 74.078, "count": 12, "max_ms": 85.106, "p95_ms": 85.106}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.439, "count": 12, "max_ms": 92.896, "p95_ms": 92.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.997, "count": 12, "max_ms": 118.474, "p95_ms": 118.474}, "read_interval": {"avg_ms": 2580.623, "count": 13, "max_ms": 16189.297, "p95_ms": 16189.297}, "triton_wait": {"avg_ms": 85.172, "count": 12, "max_ms": 104.817, "p95_ms": 104.817}, "visualize": {"avg_ms": 3.677, "count": 12, "max_ms": 4.438, "p95_ms": 4.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.456, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2210}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5979.23, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:18:33+09:00", "last_error_summary": "[h264 @ 0x55c552966780] error while decoding MB 95 19, bytestream -5 [in#0/rtsp @ 0x55c551d52e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c551d52e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:18:33+09:00", "last_error_summary": "[h264 @ 0x55c552966780] error while decoding MB 95 19, bytestream -5 [in#0/rtsp @ 0x55c551d52e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c551d52e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:18:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 398.84, "last_started_at": "2026-07-06T21:11:54+09:00", "pid": 28538, "process": {"exists": false, "pid": 28538}, "restart_count": 56, "restart_count_per_hour": 10, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.519, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 265265, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:18:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T21:18:33+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T21:18:33+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T21:18:33.338784+09:00", "mtime_epoch": 1783340313.3387837, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:18:33+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1539.359, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.743, "pushed": 0.743, "read": 0.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.137, "count": 13, "max_ms": 4.737, "p95_ms": 4.737}, "gstreamer_push": {"avg_ms": 1.25, "count": 13, "max_ms": 1.356, "p95_ms": 1.356}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.742, "count": 13, "max_ms": 82.768, "p95_ms": 82.768}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.168, "count": 13, "max_ms": 104.805, "p95_ms": 104.805}, "yolo11_640_fire_11class": {"avg_ms": 74.371, "count": 13, "max_ms": 85.106, "p95_ms": 85.106}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.101, "count": 13, "max_ms": 92.896, "p95_ms": 92.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.278, "count": 13, "max_ms": 118.474, "p95_ms": 118.474}, "read_interval": {"avg_ms": 2580.623, "count": 13, "max_ms": 16189.297, "p95_ms": 16189.297}, "triton_wait": {"avg_ms": 85.471, "count": 13, "max_ms": 104.817, "p95_ms": 104.817}, "visualize": {"avg_ms": 3.668, "count": 13, "max_ms": 4.438, "p95_ms": 4.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.504, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2210}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5979.285, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:18:33+09:00", "last_error_summary": "[h264 @ 0x55c552966780] error while decoding MB 95 19, bytestream -5 [in#0/rtsp @ 0x55c551d52e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c551d52e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:18:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 398.848, "last_started_at": "2026-07-06T21:11:54+09:00", "pid": 28538, "process": {"exists": false, "pid": 28538}, "restart_count": 56, "restart_count_per_hour": 10, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.519, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 265265, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:18:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T21:18:33+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T21:18:33+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T21:18:33.338784+09:00", "mtime_epoch": 1783340313.3387837, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:18:33+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1539.359, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.742, "pushed": 0.742, "read": 0.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.137, "count": 13, "max_ms": 4.737, "p95_ms": 4.737}, "gstreamer_push": {"avg_ms": 1.25, "count": 13, "max_ms": 1.356, "p95_ms": 1.356}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.742, "count": 13, "max_ms": 82.768, "p95_ms": 82.768}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.168, "count": 13, "max_ms": 104.805, "p95_ms": 104.805}, "yolo11_640_fire_11class": {"avg_ms": 74.371, "count": 13, "max_ms": 85.106, "p95_ms": 85.106}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.101, "count": 13, "max_ms": 92.896, "p95_ms": 92.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.278, "count": 13, "max_ms": 118.474, "p95_ms": 118.474}, "read_interval": {"avg_ms": 2580.623, "count": 13, "max_ms": 16189.297, "p95_ms": 16189.297}, "triton_wait": {"avg_ms": 85.471, "count": 13, "max_ms": 104.817, "p95_ms": 104.817}, "visualize": {"avg_ms": 3.668, "count": 13, "max_ms": 4.438, "p95_ms": 4.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.512, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2210}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6147.348, "limit_mb": 8796093022207.996}, "detail": {"pid": 28690, "restart_count": 57, "start_count": 58}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:18:33+09:00", "last_error_summary": "[h264 @ 0x55c552966780] error while decoding MB 95 19, bytestream -5 [in#0/rtsp @ 0x55c551d52e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c551d52e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:18:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:18:35+09:00", "pid": 28690, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.613, "name": "ffmpeg", "pid": 28690, "rss_mb": 206.613, "state": "S (sleeping)", "threads": 106}, "restart_count": 57, "restart_count_per_hour": 11, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.364, "sample_count": 2376, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 265265, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:18:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.06, "last_frame_pushed_at": "2026-07-06T21:18:33+09:00", "last_frame_read_age_sec": 4.164, "last_frame_read_at": "2026-07-06T21:18:33+09:00", "mjpeg": {"age_sec": 4.044, "exists": true, "mtime": "2026-07-06T21:18:33.454786+09:00", "mtime_epoch": 1783340313.454786, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101187}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:18:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1539.359, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.603, "pushed": 0.603, "read": 0.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.137, "count": 13, "max_ms": 4.737, "p95_ms": 4.737}, "gstreamer_push": {"avg_ms": 1.25, "count": 13, "max_ms": 1.356, "p95_ms": 1.356}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.742, "count": 13, "max_ms": 82.768, "p95_ms": 82.768}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.168, "count": 13, "max_ms": 104.805, "p95_ms": 104.805}, "yolo11_640_fire_11class": {"avg_ms": 74.371, "count": 13, "max_ms": 85.106, "p95_ms": 85.106}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.101, "count": 13, "max_ms": 92.896, "p95_ms": 92.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.278, "count": 13, "max_ms": 118.474, "p95_ms": 118.474}, "read_interval": {"avg_ms": 2580.623, "count": 13, "max_ms": 16189.297, "p95_ms": 16189.297}, "triton_wait": {"avg_ms": 85.471, "count": 13, "max_ms": 104.817, "p95_ms": 104.817}, "visualize": {"avg_ms": 3.668, "count": 13, "max_ms": 4.438, "p95_ms": 4.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.559, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1990}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6041.586, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bbe5cf1a80] left block unavailable for requested intra mode [h264 @ 0x55bbe5cf1a80] error while decoding MB 0 9, bytestream 2642 [h264 @ 0x55bbe5c9a600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bbe5c9a600] error while decoding MB 0 5, bytestream 3248 [h264 @ 0x55bbe5db76c0] error while decoding MB 35 7, bytestream -10 [h264 @ 0x55bbe5db76c0] error while decoding MB 67 15, bytestream -22 [h264 @ 0x55bbe5db76c0] error while decoding MB 33 23, bytestream -15 [in", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 113, "h264_decode_error_count": 370, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:20:54+09:00", "last_error_summary": "[h264 @ 0x55bbe5cf1a80] left block unavailable for requested intra mode [h264 @ 0x55bbe5cf1a80] error while decoding MB 0 9, bytestream 2642 [h264 @ 0x55bbe5c9a600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bbe5c9a600] error while decoding MB 0 5, bytestream 3248 [h264 @ 0x55bbe5db76c0] error while decoding MB 35 7, bytestream -10 [h264 @ 0x55bbe5db76c0] error while decoding MB 67 15, bytestream -22 [h264 @ 0x55bbe5db76c0] error while decoding MB 33 23, bytestream -15 [in", "last_exit_at": "2026-07-06T21:11:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 588.147, "last_started_at": "2026-07-06T21:11:06+09:00", "pid": 28431, "process": {"exists": false, "pid": 28431}, "restart_count": 122, "restart_count_per_hour": 8, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 123}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.165, "sample_count": 2613, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 238832, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:20:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T21:20:54+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T21:20:54+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-06T21:20:54.101485+09:00", "mtime_epoch": 1783340454.1014848, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130624}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:20:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1651.727, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.243, "pushed": 3.243, "read": 3.243}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.56, "count": 97, "max_ms": 5.555, "p95_ms": 4.182}, "gstreamer_push": {"avg_ms": 1.329, "count": 97, "max_ms": 1.461, "p95_ms": 1.434}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.142, "count": 97, "max_ms": 89.605, "p95_ms": 83.283}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.256, "count": 97, "max_ms": 124.387, "p95_ms": 106.597}, "yolo11_640_fire_11class": {"avg_ms": 75.766, "count": 97, "max_ms": 91.691, "p95_ms": 87.895}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.676, "count": 97, "max_ms": 101.526, "p95_ms": 98.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.841, "count": 97, "max_ms": 136.638, "p95_ms": 119.07}, "read_interval": {"avg_ms": 308.912, "count": 97, "max_ms": 19177.785, "p95_ms": 137.031}, "triton_wait": {"avg_ms": 86.443, "count": 97, "max_ms": 124.401, "p95_ms": 106.609}, "visualize": {"avg_ms": 3.678, "count": 97, "max_ms": 4.379, "p95_ms": 3.966}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1990}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6040.695, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:20:54+09:00", "last_error_summary": "[h264 @ 0x55bbe5cf1a80] left block unavailable for requested intra mode [h264 @ 0x55bbe5cf1a80] error while decoding MB 0 9, bytestream 2642 [h264 @ 0x55bbe5c9a600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bbe5c9a600] error while decoding MB 0 5, bytestream 3248 [h264 @ 0x55bbe5db76c0] error while decoding MB 35 7, bytestream -10 [h264 @ 0x55bbe5db76c0] error while decoding MB 67 15, bytestream -22 [h264 @ 0x55bbe5db76c0] error while decoding MB 33 23, bytestream -15 [in", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 113, "h264_decode_error_count": 370, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:20:54+09:00", "last_error_summary": "[h264 @ 0x55bbe5cf1a80] left block unavailable for requested intra mode [h264 @ 0x55bbe5cf1a80] error while decoding MB 0 9, bytestream 2642 [h264 @ 0x55bbe5c9a600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bbe5c9a600] error while decoding MB 0 5, bytestream 3248 [h264 @ 0x55bbe5db76c0] error while decoding MB 35 7, bytestream -10 [h264 @ 0x55bbe5db76c0] error while decoding MB 67 15, bytestream -22 [h264 @ 0x55bbe5db76c0] error while decoding MB 33 23, bytestream -15 [in", "last_exit_at": "2026-07-06T21:20:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 588.204, "last_started_at": "2026-07-06T21:11:06+09:00", "pid": 28431, "process": {"exists": false, "pid": 28431}, "restart_count": 122, "restart_count_per_hour": 8, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 123}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.165, "sample_count": 2614, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 238833, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:20:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T21:20:54+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T21:20:54+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-06T21:20:54.101485+09:00", "mtime_epoch": 1783340454.1014848, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130624}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:20:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1651.727, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.27, "pushed": 3.27, "read": 3.237}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.558, "count": 98, "max_ms": 5.555, "p95_ms": 4.182}, "gstreamer_push": {"avg_ms": 1.327, "count": 98, "max_ms": 1.461, "p95_ms": 1.434}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.074, "count": 98, "max_ms": 89.605, "p95_ms": 83.283}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.215, "count": 98, "max_ms": 124.387, "p95_ms": 106.597}, "yolo11_640_fire_11class": {"avg_ms": 75.766, "count": 98, "max_ms": 91.691, "p95_ms": 87.895}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.697, "count": 98, "max_ms": 101.526, "p95_ms": 98.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.839, "count": 98, "max_ms": 136.638, "p95_ms": 119.07}, "read_interval": {"avg_ms": 308.912, "count": 97, "max_ms": 19177.785, "p95_ms": 137.031}, "triton_wait": {"avg_ms": 86.447, "count": 98, "max_ms": 124.401, "p95_ms": 106.609}, "visualize": {"avg_ms": 3.676, "count": 98, "max_ms": 4.379, "p95_ms": 3.966}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1990}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6040.785, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 113, "h264_decode_error_count": 370, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:20:54+09:00", "last_error_summary": "[h264 @ 0x55bbe5cf1a80] left block unavailable for requested intra mode [h264 @ 0x55bbe5cf1a80] error while decoding MB 0 9, bytestream 2642 [h264 @ 0x55bbe5c9a600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bbe5c9a600] error while decoding MB 0 5, bytestream 3248 [h264 @ 0x55bbe5db76c0] error while decoding MB 35 7, bytestream -10 [h264 @ 0x55bbe5db76c0] error while decoding MB 67 15, bytestream -22 [h264 @ 0x55bbe5db76c0] error while decoding MB 33 23, bytestream -15 [in", "last_exit_at": "2026-07-06T21:20:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 588.212, "last_started_at": "2026-07-06T21:11:06+09:00", "pid": 28431, "process": {"exists": false, "pid": 28431}, "restart_count": 122, "restart_count_per_hour": 8, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 123}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.165, "sample_count": 2614, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 238833, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:20:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-06T21:20:54+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T21:20:54+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T21:20:54.101485+09:00", "mtime_epoch": 1783340454.1014848, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130624}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:20:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1651.727, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.269, "pushed": 3.269, "read": 3.236}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.558, "count": 98, "max_ms": 5.555, "p95_ms": 4.182}, "gstreamer_push": {"avg_ms": 1.327, "count": 98, "max_ms": 1.461, "p95_ms": 1.434}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.074, "count": 98, "max_ms": 89.605, "p95_ms": 83.283}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.215, "count": 98, "max_ms": 124.387, "p95_ms": 106.597}, "yolo11_640_fire_11class": {"avg_ms": 75.766, "count": 98, "max_ms": 91.691, "p95_ms": 87.895}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.697, "count": 98, "max_ms": 101.526, "p95_ms": 98.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.839, "count": 98, "max_ms": 136.638, "p95_ms": 119.07}, "read_interval": {"avg_ms": 308.912, "count": 97, "max_ms": 19177.785, "p95_ms": 137.031}, "triton_wait": {"avg_ms": 86.447, "count": 98, "max_ms": 124.401, "p95_ms": 106.609}, "visualize": {"avg_ms": 3.676, "count": 98, "max_ms": 4.379, "p95_ms": 3.966}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 1990}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6148.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 28842, "restart_count": 123, "start_count": 124}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 113, "h264_decode_error_count": 370, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:20:54+09:00", "last_error_summary": "[h264 @ 0x55bbe5cf1a80] left block unavailable for requested intra mode [h264 @ 0x55bbe5cf1a80] error while decoding MB 0 9, bytestream 2642 [h264 @ 0x55bbe5c9a600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bbe5c9a600] error while decoding MB 0 5, bytestream 3248 [h264 @ 0x55bbe5db76c0] error while decoding MB 35 7, bytestream -10 [h264 @ 0x55bbe5db76c0] error while decoding MB 67 15, bytestream -22 [h264 @ 0x55bbe5db76c0] error while decoding MB 33 23, bytestream -15 [in", "last_exit_at": "2026-07-06T21:20:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:20:56+09:00", "pid": 28842, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.277, "name": "ffmpeg", "pid": 28842, "rss_mb": 139.277, "state": "S (sleeping)", "threads": 106}, "restart_count": 123, "restart_count_per_hour": 9, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 124}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.165, "sample_count": 2577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 238833, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:20:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.045, "last_frame_pushed_at": "2026-07-06T21:20:54+09:00", "last_frame_read_age_sec": 4.145, "last_frame_read_at": "2026-07-06T21:20:54+09:00", "mjpeg": {"age_sec": 4.027, "exists": true, "mtime": "2026-07-06T21:20:54.213487+09:00", "mtime_epoch": 1783340454.213487, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129885}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:20:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1651.727, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.071, "pushed": 2.071, "read": 2.037}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.591, "count": 62, "max_ms": 5.555, "p95_ms": 4.79}, "gstreamer_push": {"avg_ms": 1.314, "count": 62, "max_ms": 1.437, "p95_ms": 1.427}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.992, "count": 62, "max_ms": 83.283, "p95_ms": 80.333}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.331, "count": 62, "max_ms": 111.352, "p95_ms": 100.615}, "yolo11_640_fire_11class": {"avg_ms": 74.916, "count": 62, "max_ms": 88.312, "p95_ms": 83.976}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.027, "count": 62, "max_ms": 101.526, "p95_ms": 96.53}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.12, "count": 62, "max_ms": 123.755, "p95_ms": 114.805}, "read_interval": {"avg_ms": 424.26, "count": 61, "max_ms": 19177.785, "p95_ms": 134.087}, "triton_wait": {"avg_ms": 85.684, "count": 62, "max_ms": 111.364, "p95_ms": 101.537}, "visualize": {"avg_ms": 3.675, "count": 62, "max_ms": 4.379, "p95_ms": 3.952}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2235}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5996.543, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55dcb48b7640] error while decoding MB 33 49, bytestream -5 [in#0/rtsp @ 0x55dcb3f0ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcb3f0ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:24:18+09:00", "last_error_summary": "[h264 @ 0x55dcb48b7640] error while decoding MB 33 49, bytestream -5 [in#0/rtsp @ 0x55dcb3f0ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcb3f0ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:18:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 342.545, "last_started_at": "2026-07-06T21:18:35+09:00", "pid": 28690, "process": {"exists": false, "pid": 28690}, "restart_count": 57, "restart_count_per_hour": 11, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.258, "sample_count": 2607, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 268240, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:24:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T21:24:17+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T21:24:17+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T21:24:17.985368+09:00", "mtime_epoch": 1783340657.9853685, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:24:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1540.906, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.202, "pushed": 3.202, "read": 3.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.236, "count": 96, "max_ms": 7.135, "p95_ms": 6.258}, "gstreamer_push": {"avg_ms": 1.269, "count": 96, "max_ms": 1.625, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.527, "count": 96, "max_ms": 85.194, "p95_ms": 79.884}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.001, "count": 96, "max_ms": 106.289, "p95_ms": 91.522}, "yolo11_640_fire_11class": {"avg_ms": 75.402, "count": 96, "max_ms": 87.591, "p95_ms": 84.653}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.647, "count": 96, "max_ms": 96.035, "p95_ms": 93.406}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.271, "count": 96, "max_ms": 119.253, "p95_ms": 108.964}, "read_interval": {"avg_ms": 312.696, "count": 96, "max_ms": 10566.506, "p95_ms": 130.264}, "triton_wait": {"avg_ms": 84.244, "count": 96, "max_ms": 106.303, "p95_ms": 96.048}, "visualize": {"avg_ms": 3.652, "count": 96, "max_ms": 5.993, "p95_ms": 4.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2235}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5995.941, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:24:18+09:00", "last_error_summary": "[h264 @ 0x55dcb48b7640] error while decoding MB 33 49, bytestream -5 [in#0/rtsp @ 0x55dcb3f0ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcb3f0ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:24:18+09:00", "last_error_summary": "[h264 @ 0x55dcb48b7640] error while decoding MB 33 49, bytestream -5 [in#0/rtsp @ 0x55dcb3f0ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcb3f0ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:24:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 342.595, "last_started_at": "2026-07-06T21:18:35+09:00", "pid": 28690, "process": {"exists": false, "pid": 28690}, "restart_count": 57, "restart_count_per_hour": 11, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.258, "sample_count": 2607, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 268241, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:24:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T21:24:18+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-06T21:24:17+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T21:24:17.985368+09:00", "mtime_epoch": 1783340657.9853685, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:24:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1540.906, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.208, "pushed": 3.208, "read": 3.175}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.222, "count": 96, "max_ms": 7.135, "p95_ms": 6.258}, "gstreamer_push": {"avg_ms": 1.271, "count": 96, "max_ms": 1.625, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.504, "count": 96, "max_ms": 85.194, "p95_ms": 79.884}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.108, "count": 96, "max_ms": 106.289, "p95_ms": 91.522}, "yolo11_640_fire_11class": {"avg_ms": 75.388, "count": 96, "max_ms": 87.591, "p95_ms": 84.653}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.687, "count": 96, "max_ms": 96.035, "p95_ms": 93.406}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.302, "count": 96, "max_ms": 119.253, "p95_ms": 108.964}, "read_interval": {"avg_ms": 314.777, "count": 95, "max_ms": 10566.506, "p95_ms": 130.264}, "triton_wait": {"avg_ms": 84.284, "count": 96, "max_ms": 106.303, "p95_ms": 96.048}, "visualize": {"avg_ms": 3.653, "count": 96, "max_ms": 5.993, "p95_ms": 4.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2235}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5996.059, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:24:18+09:00", "last_error_summary": "[h264 @ 0x55dcb48b7640] error while decoding MB 33 49, bytestream -5 [in#0/rtsp @ 0x55dcb3f0ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcb3f0ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:24:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 342.603, "last_started_at": "2026-07-06T21:18:35+09:00", "pid": 28690, "process": {"exists": false, "pid": 28690}, "restart_count": 57, "restart_count_per_hour": 11, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.258, "sample_count": 2607, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 268241, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:24:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T21:24:18+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-06T21:24:17+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-06T21:24:17.985368+09:00", "mtime_epoch": 1783340657.9853685, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:24:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1540.906, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.208, "pushed": 3.208, "read": 3.174}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.222, "count": 96, "max_ms": 7.135, "p95_ms": 6.258}, "gstreamer_push": {"avg_ms": 1.271, "count": 96, "max_ms": 1.625, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.504, "count": 96, "max_ms": 85.194, "p95_ms": 79.884}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.108, "count": 96, "max_ms": 106.289, "p95_ms": 91.522}, "yolo11_640_fire_11class": {"avg_ms": 75.388, "count": 96, "max_ms": 87.591, "p95_ms": 84.653}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.687, "count": 96, "max_ms": 96.035, "p95_ms": 93.406}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.302, "count": 96, "max_ms": 119.253, "p95_ms": 108.964}, "read_interval": {"avg_ms": 314.777, "count": 95, "max_ms": 10566.506, "p95_ms": 130.264}, "triton_wait": {"avg_ms": 84.284, "count": 96, "max_ms": 106.303, "p95_ms": 96.048}, "visualize": {"avg_ms": 3.653, "count": 96, "max_ms": 5.993, "p95_ms": 4.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2235}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6003.652, "limit_mb": 8796093022207.996}, "detail": {"pid": 28949, "restart_count": 58, "start_count": 59}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:24:18+09:00", "last_error_summary": "[h264 @ 0x55dcb48b7640] error while decoding MB 33 49, bytestream -5 [in#0/rtsp @ 0x55dcb3f0ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcb3f0ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:24:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:24:20+09:00", "pid": 28949, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.926, "name": "ffmpeg", "pid": 28949, "rss_mb": 57.926, "state": "S (sleeping)", "threads": 1}, "restart_count": 58, "restart_count_per_hour": 12, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.258, "sample_count": 2566, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 268241, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:24:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.557, "last_frame_pushed_at": "2026-07-06T21:24:18+09:00", "last_frame_read_age_sec": 4.664, "last_frame_read_at": "2026-07-06T21:24:17+09:00", "mjpeg": {"age_sec": 4.543, "exists": true, "mtime": "2026-07-06T21:24:18.101371+09:00", "mtime_epoch": 1783340658.1013708, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100833}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:24:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1540.906, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.838, "pushed": 1.838, "read": 1.804}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.152, "count": 55, "max_ms": 5.888, "p95_ms": 4.749}, "gstreamer_push": {"avg_ms": 1.283, "count": 55, "max_ms": 1.455, "p95_ms": 1.4}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.422, "count": 55, "max_ms": 80.126, "p95_ms": 78.005}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.869, "count": 55, "max_ms": 102.954, "p95_ms": 98.107}, "yolo11_640_fire_11class": {"avg_ms": 75.62, "count": 55, "max_ms": 87.591, "p95_ms": 84.849}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.077, "count": 55, "max_ms": 96.035, "p95_ms": 93.777}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.773, "count": 55, "max_ms": 116.481, "p95_ms": 111.248}, "read_interval": {"avg_ms": 469.665, "count": 54, "max_ms": 10566.506, "p95_ms": 132.515}, "triton_wait": {"avg_ms": 84.823, "count": 55, "max_ms": 102.966, "p95_ms": 98.117}, "visualize": {"avg_ms": 3.652, "count": 55, "max_ms": 4.681, "p95_ms": 4.498}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6033.461, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55cf24400040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55cf24400040] error while decoding MB 0 36, bytestream 935 [h264 @ 0x55cf23837680] error while decoding MB 76 20, bytestream -6 [in#0/rtsp @ 0x55cf23715e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cf23715e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 114, "h264_decode_error_count": 372, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:24:43+09:00", "last_error_summary": "[h264 @ 0x55cf24400040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55cf24400040] error while decoding MB 0 36, bytestream 935 [h264 @ 0x55cf23837680] error while decoding MB 76 20, bytestream -6 [in#0/rtsp @ 0x55cf23715e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cf23715e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:20:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.952, "last_started_at": "2026-07-06T21:20:56+09:00", "pid": 28842, "process": {"exists": false, "pid": 28842}, "restart_count": 123, "restart_count_per_hour": 9, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 124}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.795, "sample_count": 2377, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 240746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:24:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T21:24:43+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-06T21:24:43+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-06T21:24:43.145846+09:00", "mtime_epoch": 1783340683.145846, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120200}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:24:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1652.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.904, "pushed": 3.904, "read": 3.904}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.503, "count": 117, "max_ms": 5.606, "p95_ms": 3.96}, "gstreamer_push": {"avg_ms": 1.312, "count": 117, "max_ms": 1.504, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.065, "count": 117, "max_ms": 82.13, "p95_ms": 77.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.613, "count": 117, "max_ms": 103.04, "p95_ms": 88.875}, "yolo11_640_fire_11class": {"avg_ms": 58.91, "count": 117, "max_ms": 83.998, "p95_ms": 78.246}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.449, "count": 117, "max_ms": 92.344, "p95_ms": 88.254}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.047, "count": 117, "max_ms": 115.588, "p95_ms": 102.165}, "read_interval": {"avg_ms": 256.516, "count": 117, "max_ms": 10131.705, "p95_ms": 116.944}, "triton_wait": {"avg_ms": 65.889, "count": 117, "max_ms": 103.052, "p95_ms": 89.726}, "visualize": {"avg_ms": 3.656, "count": 117, "max_ms": 4.17, "p95_ms": 3.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6033.207, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:24:43+09:00", "last_error_summary": "[h264 @ 0x55cf24400040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55cf24400040] error while decoding MB 0 36, bytestream 935 [h264 @ 0x55cf23837680] error while decoding MB 76 20, bytestream -6 [in#0/rtsp @ 0x55cf23715e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cf23715e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 114, "h264_decode_error_count": 372, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:24:43+09:00", "last_error_summary": "[h264 @ 0x55cf24400040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55cf24400040] error while decoding MB 0 36, bytestream 935 [h264 @ 0x55cf23837680] error while decoding MB 76 20, bytestream -6 [in#0/rtsp @ 0x55cf23715e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cf23715e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:24:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.976, "last_started_at": "2026-07-06T21:20:56+09:00", "pid": 28842, "process": {"exists": false, "pid": 28842}, "restart_count": 123, "restart_count_per_hour": 9, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 124}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.795, "sample_count": 2378, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 240747, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:24:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T21:24:43+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-06T21:24:43+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-06T21:24:43.145846+09:00", "mtime_epoch": 1783340683.145846, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120200}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:24:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1652.758, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.934, "pushed": 3.934, "read": 3.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.508, "count": 118, "max_ms": 5.606, "p95_ms": 4.021}, "gstreamer_push": {"avg_ms": 1.309, "count": 118, "max_ms": 1.504, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.016, "count": 118, "max_ms": 82.13, "p95_ms": 77.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.557, "count": 118, "max_ms": 103.04, "p95_ms": 88.875}, "yolo11_640_fire_11class": {"avg_ms": 58.909, "count": 118, "max_ms": 83.998, "p95_ms": 78.246}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.421, "count": 118, "max_ms": 92.344, "p95_ms": 88.254}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.985, "count": 118, "max_ms": 115.588, "p95_ms": 102.165}, "read_interval": {"avg_ms": 256.516, "count": 117, "max_ms": 10131.705, "p95_ms": 116.944}, "triton_wait": {"avg_ms": 65.829, "count": 118, "max_ms": 103.052, "p95_ms": 89.726}, "visualize": {"avg_ms": 3.654, "count": 118, "max_ms": 4.17, "p95_ms": 3.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6034.246, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 114, "h264_decode_error_count": 372, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:24:43+09:00", "last_error_summary": "[h264 @ 0x55cf24400040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55cf24400040] error while decoding MB 0 36, bytestream 935 [h264 @ 0x55cf23837680] error while decoding MB 76 20, bytestream -6 [in#0/rtsp @ 0x55cf23715e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cf23715e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:24:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.983, "last_started_at": "2026-07-06T21:20:56+09:00", "pid": 28842, "process": {"exists": false, "pid": 28842}, "restart_count": 123, "restart_count_per_hour": 9, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 124}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.795, "sample_count": 2378, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 240747, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:24:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T21:24:43+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-06T21:24:43+09:00", "mjpeg": {"age_sec": 0.076, "exists": true, "mtime": "2026-07-06T21:24:43.145846+09:00", "mtime_epoch": 1783340683.145846, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120200}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:24:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1652.758, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.9, "pushed": 3.934, "read": 3.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.508, "count": 117, "max_ms": 5.606, "p95_ms": 4.021}, "gstreamer_push": {"avg_ms": 1.309, "count": 118, "max_ms": 1.504, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.965, "count": 117, "max_ms": 82.13, "p95_ms": 77.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.387, "count": 117, "max_ms": 103.04, "p95_ms": 88.875}, "yolo11_640_fire_11class": {"avg_ms": 58.85, "count": 117, "max_ms": 83.998, "p95_ms": 78.246}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.37, "count": 117, "max_ms": 92.344, "p95_ms": 88.254}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.823, "count": 117, "max_ms": 115.588, "p95_ms": 102.165}, "read_interval": {"avg_ms": 256.516, "count": 117, "max_ms": 10131.705, "p95_ms": 116.944}, "triton_wait": {"avg_ms": 65.669, "count": 117, "max_ms": 103.052, "p95_ms": 89.726}, "visualize": {"avg_ms": 3.653, "count": 117, "max_ms": 4.17, "p95_ms": 3.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6142.75, "limit_mb": 8796093022207.996}, "detail": {"pid": 29101, "restart_count": 124, "start_count": 125}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 114, "h264_decode_error_count": 372, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:24:43+09:00", "last_error_summary": "[h264 @ 0x55cf24400040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55cf24400040] error while decoding MB 0 36, bytestream 935 [h264 @ 0x55cf23837680] error while decoding MB 76 20, bytestream -6 [in#0/rtsp @ 0x55cf23715e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cf23715e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:24:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:24:46+09:00", "pid": 29101, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.332, "name": "ffmpeg", "pid": 29101, "rss_mb": 139.332, "state": "S (sleeping)", "threads": 106}, "restart_count": 124, "restart_count_per_hour": 10, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 125}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.795, "sample_count": 2333, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 240747, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:24:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.979, "last_frame_pushed_at": "2026-07-06T21:24:43+09:00", "last_frame_read_age_sec": 5.051, "last_frame_read_at": "2026-07-06T21:24:43+09:00", "mjpeg": {"age_sec": 4.963, "exists": true, "mtime": "2026-07-06T21:24:43.225847+09:00", "mtime_epoch": 1783340683.2258475, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120112}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:24:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1652.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.24, "pushed": 2.24, "read": 2.206}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.543, "count": 67, "max_ms": 5.606, "p95_ms": 4.123}, "gstreamer_push": {"avg_ms": 1.291, "count": 67, "max_ms": 1.414, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.985, "count": 67, "max_ms": 80.899, "p95_ms": 70.612}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.895, "count": 67, "max_ms": 90.54, "p95_ms": 80.919}, "yolo11_640_fire_11class": {"avg_ms": 55.604, "count": 67, "max_ms": 80.91, "p95_ms": 74.372}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.588, "count": 67, "max_ms": 89.714, "p95_ms": 75.079}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.988, "count": 67, "max_ms": 102.694, "p95_ms": 97.996}, "read_interval": {"avg_ms": 378.134, "count": 66, "max_ms": 10131.705, "p95_ms": 112.965}, "triton_wait": {"avg_ms": 59.913, "count": 67, "max_ms": 90.552, "p95_ms": 85.918}, "visualize": {"avg_ms": 3.637, "count": 67, "max_ms": 4.162, "p95_ms": 3.856}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2248}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5994.551, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564309591540] error while decoding MB 60 28, bytestream -11 [in#0/rtsp @ 0x564308ff9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564308ff9e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:27:41+09:00", "last_error_summary": "[h264 @ 0x564309591540] error while decoding MB 60 28, bytestream -11 [in#0/rtsp @ 0x564308ff9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564308ff9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:24:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 201.147, "last_started_at": "2026-07-06T21:24:20+09:00", "pid": 28949, "process": {"exists": false, "pid": 28949}, "restart_count": 58, "restart_count_per_hour": 12, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.071, "sample_count": 2224, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 269773, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:27:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-06T21:27:41+09:00", "last_frame_read_age_sec": 0.071, "last_frame_read_at": "2026-07-06T21:27:41+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-06T21:27:41.725226+09:00", "mtime_epoch": 1783340861.7252257, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97970}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:27:41+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1541.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 40, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.92, "pushed": 1.92, "read": 1.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.504, "count": 39, "max_ms": 6.963, "p95_ms": 6.917}, "gstreamer_push": {"avg_ms": 1.331, "count": 39, "max_ms": 1.809, "p95_ms": 1.723}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.926, "count": 39, "max_ms": 84.644, "p95_ms": 84.042}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.21, "count": 39, "max_ms": 119.277, "p95_ms": 110.412}, "yolo11_640_fire_11class": {"avg_ms": 75.973, "count": 39, "max_ms": 86.137, "p95_ms": 84.566}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.592, "count": 39, "max_ms": 96.242, "p95_ms": 96.227}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.87, "count": 39, "max_ms": 140.034, "p95_ms": 123.146}, "read_interval": {"avg_ms": 895.242, "count": 40, "max_ms": 16071.105, "p95_ms": 152.783}, "triton_wait": {"avg_ms": 85.136, "count": 39, "max_ms": 119.292, "p95_ms": 110.424}, "visualize": {"avg_ms": 3.91, "count": 39, "max_ms": 6.152, "p95_ms": 6.064}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 20.315, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2248}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5982.137, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:27:41+09:00", "last_error_summary": "[h264 @ 0x564309591540] error while decoding MB 60 28, bytestream -11 [in#0/rtsp @ 0x564308ff9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564308ff9e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:27:41+09:00", "last_error_summary": "[h264 @ 0x564309591540] error while decoding MB 60 28, bytestream -11 [in#0/rtsp @ 0x564308ff9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564308ff9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:27:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 201.184, "last_started_at": "2026-07-06T21:24:20+09:00", "pid": 28949, "process": {"exists": false, "pid": 28949}, "restart_count": 58, "restart_count_per_hour": 12, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.071, "sample_count": 2224, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 269774, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:27:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T21:27:41+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T21:27:41+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T21:27:41.725226+09:00", "mtime_epoch": 1783340861.7252257, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97970}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:27:41+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1541.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 40, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.965, "pushed": 1.965, "read": 1.965}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.506, "count": 40, "max_ms": 6.963, "p95_ms": 6.902}, "gstreamer_push": {"avg_ms": 1.328, "count": 40, "max_ms": 1.809, "p95_ms": 1.674}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.325, "count": 40, "max_ms": 84.644, "p95_ms": 81.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.178, "count": 40, "max_ms": 119.277, "p95_ms": 103.306}, "yolo11_640_fire_11class": {"avg_ms": 76.071, "count": 40, "max_ms": 86.137, "p95_ms": 84.274}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.696, "count": 40, "max_ms": 96.242, "p95_ms": 95.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.921, "count": 40, "max_ms": 140.034, "p95_ms": 116.893}, "read_interval": {"avg_ms": 895.242, "count": 40, "max_ms": 16071.105, "p95_ms": 152.783}, "triton_wait": {"avg_ms": 85.202, "count": 40, "max_ms": 119.292, "p95_ms": 103.319}, "visualize": {"avg_ms": 3.905, "count": 40, "max_ms": 6.152, "p95_ms": 6.02}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 20.352, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2248}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5982.23, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:27:41+09:00", "last_error_summary": "[h264 @ 0x564309591540] error while decoding MB 60 28, bytestream -11 [in#0/rtsp @ 0x564308ff9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564308ff9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:27:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 201.193, "last_started_at": "2026-07-06T21:24:20+09:00", "pid": 28949, "process": {"exists": false, "pid": 28949}, "restart_count": 58, "restart_count_per_hour": 12, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.071, "sample_count": 2224, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 269774, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:27:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T21:27:41+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T21:27:41+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T21:27:41.725226+09:00", "mtime_epoch": 1783340861.7252257, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97970}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:27:41+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1541.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 40, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.965, "pushed": 1.965, "read": 1.965}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.506, "count": 40, "max_ms": 6.963, "p95_ms": 6.902}, "gstreamer_push": {"avg_ms": 1.328, "count": 40, "max_ms": 1.809, "p95_ms": 1.674}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.325, "count": 40, "max_ms": 84.644, "p95_ms": 81.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.178, "count": 40, "max_ms": 119.277, "p95_ms": 103.306}, "yolo11_640_fire_11class": {"avg_ms": 76.071, "count": 40, "max_ms": 86.137, "p95_ms": 84.274}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.696, "count": 40, "max_ms": 96.242, "p95_ms": 95.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.921, "count": 40, "max_ms": 140.034, "p95_ms": 116.893}, "read_interval": {"avg_ms": 895.242, "count": 40, "max_ms": 16071.105, "p95_ms": 152.783}, "triton_wait": {"avg_ms": 85.202, "count": 40, "max_ms": 119.292, "p95_ms": 103.319}, "visualize": {"avg_ms": 3.905, "count": 40, "max_ms": 6.152, "p95_ms": 6.02}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 20.361, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2248}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6009.289, "limit_mb": 8796093022207.996}, "detail": {"pid": 29208, "restart_count": 59, "start_count": 60}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:27:41+09:00", "last_error_summary": "[h264 @ 0x564309591540] error while decoding MB 60 28, bytestream -11 [in#0/rtsp @ 0x564308ff9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564308ff9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:27:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:27:44+09:00", "pid": 29208, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.223, "name": "ffmpeg", "pid": 29208, "rss_mb": 58.223, "state": "S (sleeping)", "threads": 1}, "restart_count": 59, "restart_count_per_hour": 13, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.181, "sample_count": 2180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 269774, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:27:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 5.006, "last_frame_pushed_at": "2026-07-06T21:27:41+09:00", "last_frame_read_age_sec": 5.109, "last_frame_read_at": "2026-07-06T21:27:41+09:00", "mjpeg": {"age_sec": 4.987, "exists": true, "mtime": "2026-07-06T21:27:41.841228+09:00", "mtime_epoch": 1783340861.8412278, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96200}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:27:46+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1541.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 40, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.578, "pushed": 1.578, "read": 1.578}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.506, "count": 40, "max_ms": 6.963, "p95_ms": 6.902}, "gstreamer_push": {"avg_ms": 1.328, "count": 40, "max_ms": 1.809, "p95_ms": 1.674}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.325, "count": 40, "max_ms": 84.644, "p95_ms": 81.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.178, "count": 40, "max_ms": 119.277, "p95_ms": 103.306}, "yolo11_640_fire_11class": {"avg_ms": 76.071, "count": 40, "max_ms": 86.137, "p95_ms": 84.274}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.696, "count": 40, "max_ms": 96.242, "p95_ms": 95.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.921, "count": 40, "max_ms": 140.034, "p95_ms": 116.893}, "read_interval": {"avg_ms": 895.242, "count": 40, "max_ms": 16071.105, "p95_ms": 152.783}, "triton_wait": {"avg_ms": 85.202, "count": 40, "max_ms": 119.292, "p95_ms": 103.319}, "visualize": {"avg_ms": 3.905, "count": 40, "max_ms": 6.152, "p95_ms": 6.02}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 25.353, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2257}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6007.004, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559ca40a8d40] error while decoding MB 13 62, bytestream -15 [in#0/rtsp @ 0x559ca4066e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ca4066e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:30:20+09:00", "last_error_summary": "[h264 @ 0x559ca40a8d40] error while decoding MB 13 62, bytestream -15 [in#0/rtsp @ 0x559ca4066e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ca4066e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:27:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.353, "last_started_at": "2026-07-06T21:27:44+09:00", "pid": 29208, "process": {"exists": false, "pid": 29208}, "restart_count": 59, "restart_count_per_hour": 13, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.181, "sample_count": 2086, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:30:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T21:30:20+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T21:30:20+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T21:30:20.124213+09:00", "mtime_epoch": 1783341020.1242132, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102852}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:30:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1542.195, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.754, "pushed": 0.754, "read": 0.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.974, "count": 13, "max_ms": 4.166, "p95_ms": 4.166}, "gstreamer_push": {"avg_ms": 1.257, "count": 13, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.533, "count": 13, "max_ms": 88.431, "p95_ms": 88.431}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.323, "count": 13, "max_ms": 87.903, "p95_ms": 87.903}, "yolo11_640_fire_11class": {"avg_ms": 67.729, "count": 13, "max_ms": 89.886, "p95_ms": 89.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.567, "count": 13, "max_ms": 90.391, "p95_ms": 90.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.111, "count": 13, "max_ms": 103.273, "p95_ms": 103.273}, "read_interval": {"avg_ms": 2253.129, "count": 14, "max_ms": 16016.525, "p95_ms": 16016.525}, "triton_wait": {"avg_ms": 72.689, "count": 13, "max_ms": 90.401, "p95_ms": 90.401}, "visualize": {"avg_ms": 3.556, "count": 13, "max_ms": 3.833, "p95_ms": 3.833}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.241, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2257}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6006.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:30:20+09:00", "last_error_summary": "[h264 @ 0x559ca40a8d40] error while decoding MB 13 62, bytestream -15 [in#0/rtsp @ 0x559ca4066e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ca4066e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:30:20+09:00", "last_error_summary": "[h264 @ 0x559ca40a8d40] error while decoding MB 13 62, bytestream -15 [in#0/rtsp @ 0x559ca4066e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ca4066e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:30:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.396, "last_started_at": "2026-07-06T21:27:44+09:00", "pid": 29208, "process": {"exists": false, "pid": 29208}, "restart_count": 59, "restart_count_per_hour": 13, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.181, "sample_count": 2086, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270874, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:30:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T21:30:20+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-06T21:30:20+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-06T21:30:20.124213+09:00", "mtime_epoch": 1783341020.1242132, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102852}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:30:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1542.195, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.81, "pushed": 0.81, "read": 0.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.979, "count": 14, "max_ms": 4.166, "p95_ms": 4.166}, "gstreamer_push": {"avg_ms": 1.257, "count": 14, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.665, "count": 14, "max_ms": 88.431, "p95_ms": 88.431}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.659, "count": 14, "max_ms": 87.903, "p95_ms": 87.903}, "yolo11_640_fire_11class": {"avg_ms": 68.352, "count": 14, "max_ms": 89.886, "p95_ms": 89.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.435, "count": 14, "max_ms": 90.391, "p95_ms": 90.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.96, "count": 14, "max_ms": 103.273, "p95_ms": 103.273}, "read_interval": {"avg_ms": 2253.129, "count": 14, "max_ms": 16016.525, "p95_ms": 16016.525}, "triton_wait": {"avg_ms": 73.549, "count": 14, "max_ms": 90.401, "p95_ms": 90.401}, "visualize": {"avg_ms": 3.552, "count": 14, "max_ms": 3.833, "p95_ms": 3.833}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.284, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2257}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6006.395, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:30:20+09:00", "last_error_summary": "[h264 @ 0x559ca40a8d40] error while decoding MB 13 62, bytestream -15 [in#0/rtsp @ 0x559ca4066e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ca4066e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:30:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.403, "last_started_at": "2026-07-06T21:27:44+09:00", "pid": 29208, "process": {"exists": false, "pid": 29208}, "restart_count": 59, "restart_count_per_hour": 13, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.181, "sample_count": 2086, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270874, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:30:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T21:30:20+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T21:30:20+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T21:30:20.124213+09:00", "mtime_epoch": 1783341020.1242132, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102852}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:30:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1542.195, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.81, "pushed": 0.81, "read": 0.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.979, "count": 14, "max_ms": 4.166, "p95_ms": 4.166}, "gstreamer_push": {"avg_ms": 1.257, "count": 14, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.665, "count": 14, "max_ms": 88.431, "p95_ms": 88.431}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.659, "count": 14, "max_ms": 87.903, "p95_ms": 87.903}, "yolo11_640_fire_11class": {"avg_ms": 68.352, "count": 14, "max_ms": 89.886, "p95_ms": 89.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.435, "count": 14, "max_ms": 90.391, "p95_ms": 90.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.96, "count": 14, "max_ms": 103.273, "p95_ms": 103.273}, "read_interval": {"avg_ms": 2253.129, "count": 14, "max_ms": 16016.525, "p95_ms": 16016.525}, "triton_wait": {"avg_ms": 73.549, "count": 14, "max_ms": 90.401, "p95_ms": 90.401}, "visualize": {"avg_ms": 3.552, "count": 14, "max_ms": 3.833, "p95_ms": 3.833}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.291, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2257}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6130.375, "limit_mb": 8796093022207.996}, "detail": {"pid": 29360, "restart_count": 60, "start_count": 61}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:30:20+09:00", "last_error_summary": "[h264 @ 0x559ca40a8d40] error while decoding MB 13 62, bytestream -15 [in#0/rtsp @ 0x559ca4066e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ca4066e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:30:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:30:23+09:00", "pid": 29360, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.918, "name": "ffmpeg", "pid": 29360, "rss_mb": 156.754, "state": "R (running)", "threads": 49}, "restart_count": 60, "restart_count_per_hour": 14, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.152, "sample_count": 2043, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270874, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:30:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.901, "last_frame_pushed_at": "2026-07-06T21:30:20+09:00", "last_frame_read_age_sec": 5.0, "last_frame_read_at": "2026-07-06T21:30:20+09:00", "mjpeg": {"age_sec": 4.886, "exists": true, "mtime": "2026-07-06T21:30:20.232215+09:00", "mtime_epoch": 1783341020.2322152, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:30:25+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1542.195, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.631, "pushed": 0.631, "read": 0.631}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.979, "count": 14, "max_ms": 4.166, "p95_ms": 4.166}, "gstreamer_push": {"avg_ms": 1.257, "count": 14, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.665, "count": 14, "max_ms": 88.431, "p95_ms": 88.431}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.659, "count": 14, "max_ms": 87.903, "p95_ms": 87.903}, "yolo11_640_fire_11class": {"avg_ms": 68.352, "count": 14, "max_ms": 89.886, "p95_ms": 89.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.435, "count": 14, "max_ms": 90.391, "p95_ms": 90.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.96, "count": 14, "max_ms": 103.273, "p95_ms": 103.273}, "read_interval": {"avg_ms": 2253.129, "count": 14, "max_ms": 16016.525, "p95_ms": 16016.525}, "triton_wait": {"avg_ms": 73.549, "count": 14, "max_ms": 90.401, "p95_ms": 90.401}, "visualize": {"avg_ms": 3.552, "count": 14, "max_ms": 3.833, "p95_ms": 3.833}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.179, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6049.312, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564157841380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564157841380] error while decoding MB 0 30, bytestream 3536 [h264 @ 0x5641584a2980] error while decoding MB 44 7, bytestream -18 [h264 @ 0x5641578dcc00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5641578dcc00] error while decoding MB 0 17, bytestream 2689 [h264 @ 0x5641584a2980] error while decoding MB 3 7, bytestream -5 [in#0/rtsp @ 0x5641577bae00] Error during demuxing: Connection timed", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 115, "h264_decode_error_count": 376, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:30:54+09:00", "last_error_summary": "[h264 @ 0x564157841380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564157841380] error while decoding MB 0 30, bytestream 3536 [h264 @ 0x5641584a2980] error while decoding MB 44 7, bytestream -18 [h264 @ 0x5641578dcc00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5641578dcc00] error while decoding MB 0 17, bytestream 2689 [h264 @ 0x5641584a2980] error while decoding MB 3 7, bytestream -5 [in#0/rtsp @ 0x5641577bae00] Error during demuxing: Connection timed", "last_exit_at": "2026-07-06T21:24:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 368.722, "last_started_at": "2026-07-06T21:24:46+09:00", "pid": 29101, "process": {"exists": false, "pid": 29101}, "restart_count": 124, "restart_count_per_hour": 9, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 125}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.363, "sample_count": 2741, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 244091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:30:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T21:30:54+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-06T21:30:54+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-06T21:30:54.848867+09:00", "mtime_epoch": 1783341054.848867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116828}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:30:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1654.305, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 106, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.535, "pushed": 3.535, "read": 3.535}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.591, "count": 106, "max_ms": 5.764, "p95_ms": 4.318}, "gstreamer_push": {"avg_ms": 1.343, "count": 106, "max_ms": 2.041, "p95_ms": 1.505}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.026, "count": 106, "max_ms": 88.052, "p95_ms": 80.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.782, "count": 106, "max_ms": 109.062, "p95_ms": 90.197}, "yolo11_640_fire_11class": {"avg_ms": 68.967, "count": 106, "max_ms": 86.245, "p95_ms": 80.485}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.268, "count": 106, "max_ms": 97.709, "p95_ms": 88.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.039, "count": 106, "max_ms": 121.345, "p95_ms": 107.599}, "read_interval": {"avg_ms": 282.918, "count": 106, "max_ms": 19163.107, "p95_ms": 127.385}, "triton_wait": {"avg_ms": 76.675, "count": 106, "max_ms": 109.074, "p95_ms": 95.142}, "visualize": {"avg_ms": 3.692, "count": 106, "max_ms": 6.029, "p95_ms": 3.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6048.938, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:30:54+09:00", "last_error_summary": "[h264 @ 0x564157841380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564157841380] error while decoding MB 0 30, bytestream 3536 [h264 @ 0x5641584a2980] error while decoding MB 44 7, bytestream -18 [h264 @ 0x5641578dcc00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5641578dcc00] error while decoding MB 0 17, bytestream 2689 [h264 @ 0x5641584a2980] error while decoding MB 3 7, bytestream -5 [in#0/rtsp @ 0x5641577bae00] Error during demuxing: Connection timed", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 115, "h264_decode_error_count": 376, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:30:54+09:00", "last_error_summary": "[h264 @ 0x564157841380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564157841380] error while decoding MB 0 30, bytestream 3536 [h264 @ 0x5641584a2980] error while decoding MB 44 7, bytestream -18 [h264 @ 0x5641578dcc00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5641578dcc00] error while decoding MB 0 17, bytestream 2689 [h264 @ 0x5641584a2980] error while decoding MB 3 7, bytestream -5 [in#0/rtsp @ 0x5641577bae00] Error during demuxing: Connection timed", "last_exit_at": "2026-07-06T21:30:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 368.767, "last_started_at": "2026-07-06T21:24:46+09:00", "pid": 29101, "process": {"exists": false, "pid": 29101}, "restart_count": 124, "restart_count_per_hour": 9, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 125}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.363, "sample_count": 2742, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 244092, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:30:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T21:30:54+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T21:30:54+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T21:30:54.848867+09:00", "mtime_epoch": 1783341054.848867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116828}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:30:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1654.305, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.538, "pushed": 3.538, "read": 3.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.59, "count": 106, "max_ms": 5.764, "p95_ms": 4.318}, "gstreamer_push": {"avg_ms": 1.343, "count": 106, "max_ms": 2.041, "p95_ms": 1.505}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.258, "count": 106, "max_ms": 88.052, "p95_ms": 80.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.995, "count": 106, "max_ms": 109.062, "p95_ms": 90.197}, "yolo11_640_fire_11class": {"avg_ms": 69.167, "count": 106, "max_ms": 86.245, "p95_ms": 80.485}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.544, "count": 106, "max_ms": 97.709, "p95_ms": 88.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.311, "count": 106, "max_ms": 121.345, "p95_ms": 107.599}, "read_interval": {"avg_ms": 284.825, "count": 105, "max_ms": 19163.107, "p95_ms": 127.385}, "triton_wait": {"avg_ms": 76.952, "count": 106, "max_ms": 109.074, "p95_ms": 95.142}, "visualize": {"avg_ms": 3.691, "count": 106, "max_ms": 6.029, "p95_ms": 3.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6048.965, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 115, "h264_decode_error_count": 376, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:30:54+09:00", "last_error_summary": "[h264 @ 0x564157841380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564157841380] error while decoding MB 0 30, bytestream 3536 [h264 @ 0x5641584a2980] error while decoding MB 44 7, bytestream -18 [h264 @ 0x5641578dcc00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5641578dcc00] error while decoding MB 0 17, bytestream 2689 [h264 @ 0x5641584a2980] error while decoding MB 3 7, bytestream -5 [in#0/rtsp @ 0x5641577bae00] Error during demuxing: Connection timed", "last_exit_at": "2026-07-06T21:30:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 368.776, "last_started_at": "2026-07-06T21:24:46+09:00", "pid": 29101, "process": {"exists": false, "pid": 29101}, "restart_count": 124, "restart_count_per_hour": 9, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 125}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.363, "sample_count": 2742, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 244092, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:30:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-06T21:30:54+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-06T21:30:54+09:00", "mjpeg": {"age_sec": 0.007, "exists": true, "mtime": "2026-07-06T21:30:54.956869+09:00", "mtime_epoch": 1783341054.9568691, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115161}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:30:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1654.305, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.537, "pushed": 3.537, "read": 3.504}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.59, "count": 106, "max_ms": 5.764, "p95_ms": 4.318}, "gstreamer_push": {"avg_ms": 1.343, "count": 106, "max_ms": 2.041, "p95_ms": 1.505}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.258, "count": 106, "max_ms": 88.052, "p95_ms": 80.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.995, "count": 106, "max_ms": 109.062, "p95_ms": 90.197}, "yolo11_640_fire_11class": {"avg_ms": 69.167, "count": 106, "max_ms": 86.245, "p95_ms": 80.485}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.544, "count": 106, "max_ms": 97.709, "p95_ms": 88.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.311, "count": 106, "max_ms": 121.345, "p95_ms": 107.599}, "read_interval": {"avg_ms": 284.825, "count": 105, "max_ms": 19163.107, "p95_ms": 127.385}, "triton_wait": {"avg_ms": 76.952, "count": 106, "max_ms": 109.074, "p95_ms": 95.142}, "visualize": {"avg_ms": 3.691, "count": 106, "max_ms": 6.029, "p95_ms": 3.911}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6156.035, "limit_mb": 8796093022207.996}, "detail": {"pid": 29512, "restart_count": 125, "start_count": 126}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 115, "h264_decode_error_count": 376, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:30:54+09:00", "last_error_summary": "[h264 @ 0x564157841380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564157841380] error while decoding MB 0 30, bytestream 3536 [h264 @ 0x5641584a2980] error while decoding MB 44 7, bytestream -18 [h264 @ 0x5641578dcc00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5641578dcc00] error while decoding MB 0 17, bytestream 2689 [h264 @ 0x5641584a2980] error while decoding MB 3 7, bytestream -5 [in#0/rtsp @ 0x5641577bae00] Error during demuxing: Connection timed", "last_exit_at": "2026-07-06T21:30:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:30:57+09:00", "pid": 29512, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.473, "name": "ffmpeg", "pid": 29512, "rss_mb": 138.473, "state": "S (sleeping)", "threads": 106}, "restart_count": 125, "restart_count_per_hour": 10, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 126}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.331, "sample_count": 2700, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 244092, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:30:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.75, "last_frame_pushed_at": "2026-07-06T21:30:54+09:00", "last_frame_read_age_sec": 4.851, "last_frame_read_at": "2026-07-06T21:30:54+09:00", "mjpeg": {"age_sec": 4.737, "exists": true, "mtime": "2026-07-06T21:30:54.956869+09:00", "mtime_epoch": 1783341054.9568691, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115161}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:30:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1654.305, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.938, "pushed": 1.938, "read": 1.904}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.597, "count": 58, "max_ms": 5.764, "p95_ms": 5.112}, "gstreamer_push": {"avg_ms": 1.356, "count": 58, "max_ms": 2.041, "p95_ms": 1.507}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.321, "count": 58, "max_ms": 81.27, "p95_ms": 79.338}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.071, "count": 58, "max_ms": 109.062, "p95_ms": 90.197}, "yolo11_640_fire_11class": {"avg_ms": 72.282, "count": 58, "max_ms": 85.85, "p95_ms": 81.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.275, "count": 58, "max_ms": 94.188, "p95_ms": 90.082}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.32, "count": 58, "max_ms": 121.345, "p95_ms": 106.577}, "read_interval": {"avg_ms": 441.754, "count": 57, "max_ms": 19163.107, "p95_ms": 133.235}, "triton_wait": {"avg_ms": 80.92, "count": 58, "max_ms": 109.074, "p95_ms": 94.2}, "visualize": {"avg_ms": 3.698, "count": 58, "max_ms": 6.029, "p95_ms": 4.348}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2268}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5993.422, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5641cbc9b140] error while decoding MB 101 36, bytestream -7 [in#0/rtsp @ 0x5641cb707e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5641cb707e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:33:28+09:00", "last_error_summary": "[h264 @ 0x5641cbc9b140] error while decoding MB 101 36, bytestream -7 [in#0/rtsp @ 0x5641cb707e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5641cb707e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:30:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.988, "last_started_at": "2026-07-06T21:30:23+09:00", "pid": 29360, "process": {"exists": false, "pid": 29360}, "restart_count": 60, "restart_count_per_hour": 13, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.111, "sample_count": 2124, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 272263, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:33:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T21:33:28+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T21:33:28+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T21:33:28.055482+09:00", "mtime_epoch": 1783341208.0554817, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102991}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:33:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1542.711, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.743, "pushed": 0.743, "read": 0.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.585, "count": 13, "max_ms": 5.667, "p95_ms": 5.667}, "gstreamer_push": {"avg_ms": 1.286, "count": 13, "max_ms": 1.474, "p95_ms": 1.474}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.574, "count": 13, "max_ms": 74.247, "p95_ms": 74.247}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.814, "count": 13, "max_ms": 83.599, "p95_ms": 83.599}, "yolo11_640_fire_11class": {"avg_ms": 67.712, "count": 13, "max_ms": 75.525, "p95_ms": 75.525}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.57, "count": 13, "max_ms": 89.319, "p95_ms": 89.319}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.172, "count": 13, "max_ms": 102.135, "p95_ms": 102.135}, "read_interval": {"avg_ms": 2440.264, "count": 14, "max_ms": 16726.731, "p95_ms": 16726.731}, "triton_wait": {"avg_ms": 76.904, "count": 13, "max_ms": 89.33, "p95_ms": 89.33}, "visualize": {"avg_ms": 3.651, "count": 13, "max_ms": 3.876, "p95_ms": 3.876}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.499, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2268}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5992.336, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:33:28+09:00", "last_error_summary": "[h264 @ 0x5641cbc9b140] error while decoding MB 101 36, bytestream -7 [in#0/rtsp @ 0x5641cb707e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5641cb707e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:33:28+09:00", "last_error_summary": "[h264 @ 0x5641cbc9b140] error while decoding MB 101 36, bytestream -7 [in#0/rtsp @ 0x5641cb707e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5641cb707e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:33:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.043, "last_started_at": "2026-07-06T21:30:23+09:00", "pid": 29360, "process": {"exists": false, "pid": 29360}, "restart_count": 60, "restart_count_per_hour": 13, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.111, "sample_count": 2125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 272264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:33:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T21:33:28+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T21:33:28+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-06T21:33:28.055482+09:00", "mtime_epoch": 1783341208.0554817, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102991}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:33:28+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1542.711, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.798, "pushed": 0.798, "read": 0.798}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.552, "count": 14, "max_ms": 5.667, "p95_ms": 5.667}, "gstreamer_push": {"avg_ms": 1.29, "count": 14, "max_ms": 1.474, "p95_ms": 1.474}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.031, "count": 14, "max_ms": 82.971, "p95_ms": 82.971}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.591, "count": 14, "max_ms": 95.689, "p95_ms": 95.689}, "yolo11_640_fire_11class": {"avg_ms": 68.801, "count": 14, "max_ms": 82.959, "p95_ms": 82.959}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.749, "count": 14, "max_ms": 93.078, "p95_ms": 93.078}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.501, "count": 14, "max_ms": 108.779, "p95_ms": 108.779}, "read_interval": {"avg_ms": 2440.264, "count": 14, "max_ms": 16726.731, "p95_ms": 16726.731}, "triton_wait": {"avg_ms": 78.247, "count": 14, "max_ms": 95.701, "p95_ms": 95.701}, "visualize": {"avg_ms": 3.658, "count": 14, "max_ms": 3.876, "p95_ms": 3.876}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.554, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2268}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5992.383, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:33:28+09:00", "last_error_summary": "[h264 @ 0x5641cbc9b140] error while decoding MB 101 36, bytestream -7 [in#0/rtsp @ 0x5641cb707e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5641cb707e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:33:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.05, "last_started_at": "2026-07-06T21:30:23+09:00", "pid": 29360, "process": {"exists": false, "pid": 29360}, "restart_count": 60, "restart_count_per_hour": 13, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.111, "sample_count": 2125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 272264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:33:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T21:33:28+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-06T21:33:28+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T21:33:28.055482+09:00", "mtime_epoch": 1783341208.0554817, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102991}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:33:28+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1542.711, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.797, "pushed": 0.797, "read": 0.797}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.552, "count": 14, "max_ms": 5.667, "p95_ms": 5.667}, "gstreamer_push": {"avg_ms": 1.29, "count": 14, "max_ms": 1.474, "p95_ms": 1.474}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.031, "count": 14, "max_ms": 82.971, "p95_ms": 82.971}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.591, "count": 14, "max_ms": 95.689, "p95_ms": 95.689}, "yolo11_640_fire_11class": {"avg_ms": 68.801, "count": 14, "max_ms": 82.959, "p95_ms": 82.959}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.749, "count": 14, "max_ms": 93.078, "p95_ms": 93.078}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.501, "count": 14, "max_ms": 108.779, "p95_ms": 108.779}, "read_interval": {"avg_ms": 2440.264, "count": 14, "max_ms": 16726.731, "p95_ms": 16726.731}, "triton_wait": {"avg_ms": 78.247, "count": 14, "max_ms": 95.701, "p95_ms": 95.701}, "visualize": {"avg_ms": 3.658, "count": 14, "max_ms": 3.876, "p95_ms": 3.876}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.561, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2268}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6010.453, "limit_mb": 8796093022207.996}, "detail": {"pid": 29619, "restart_count": 61, "start_count": 62}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:33:28+09:00", "last_error_summary": "[h264 @ 0x5641cbc9b140] error while decoding MB 101 36, bytestream -7 [in#0/rtsp @ 0x5641cb707e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5641cb707e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:33:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:33:30+09:00", "pid": 29619, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.07, "name": "ffmpeg", "pid": 29619, "rss_mb": 57.07, "state": "S (sleeping)", "threads": 1}, "restart_count": 61, "restart_count_per_hour": 14, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.05, "sample_count": 2084, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 272264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:33:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.65, "last_frame_pushed_at": "2026-07-06T21:33:28+09:00", "last_frame_read_age_sec": 4.761, "last_frame_read_at": "2026-07-06T21:33:28+09:00", "mjpeg": {"age_sec": 4.634, "exists": true, "mtime": "2026-07-06T21:33:28.171484+09:00", "mtime_epoch": 1783341208.1714835, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97646}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:33:32+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1542.711, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.631, "pushed": 0.631, "read": 0.631}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.552, "count": 14, "max_ms": 5.667, "p95_ms": 5.667}, "gstreamer_push": {"avg_ms": 1.29, "count": 14, "max_ms": 1.474, "p95_ms": 1.474}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.031, "count": 14, "max_ms": 82.971, "p95_ms": 82.971}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.591, "count": 14, "max_ms": 95.689, "p95_ms": 95.689}, "yolo11_640_fire_11class": {"avg_ms": 68.801, "count": 14, "max_ms": 82.959, "p95_ms": 82.959}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.749, "count": 14, "max_ms": 93.078, "p95_ms": 93.078}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.501, "count": 14, "max_ms": 108.779, "p95_ms": 108.779}, "read_interval": {"avg_ms": 2440.264, "count": 14, "max_ms": 16726.731, "p95_ms": 16726.731}, "triton_wait": {"avg_ms": 78.247, "count": 14, "max_ms": 95.701, "p95_ms": 95.701}, "visualize": {"avg_ms": 3.658, "count": 14, "max_ms": 3.876, "p95_ms": 3.876}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.198, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2060}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6045.613, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5555d2890480] reference picture missing during reorder [h264 @ 0x5555d2890480] Missing reference picture, default is 65578 [h264 @ 0x5555d295f300] mmco: unref short failure [h264 @ 0x5555d295f300] illegal short term buffer state detected [h264 @ 0x5555d295f300] error while decoding MB 79 39, bytestream -12 [h264 @ 0x5555d1e45cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5555d1e45cc0] error while decoding MB 0 17, bytestream 7474 [h264 @ 0x5555d1e9ab40] top block ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 116, "h264_decode_error_count": 379, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:36:56+09:00", "last_error_summary": "[h264 @ 0x5555d2890480] reference picture missing during reorder [h264 @ 0x5555d2890480] Missing reference picture, default is 65578 [h264 @ 0x5555d295f300] mmco: unref short failure [h264 @ 0x5555d295f300] illegal short term buffer state detected [h264 @ 0x5555d295f300] error while decoding MB 79 39, bytestream -12 [h264 @ 0x5555d1e45cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5555d1e45cc0] error while decoding MB 0 17, bytestream 7474 [h264 @ 0x5555d1e9ab40] top block ", "last_exit_at": "2026-07-06T21:30:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 358.825, "last_started_at": "2026-07-06T21:30:57+09:00", "pid": 29512, "process": {"exists": false, "pid": 29512}, "restart_count": 125, "restart_count_per_hour": 8, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 126}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.291, "sample_count": 2595, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 247202, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:36:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-06T21:36:56+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-06T21:36:56+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-06T21:36:56.455025+09:00", "mtime_epoch": 1783341416.4550252, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126522}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:36:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1655.852, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.267, "pushed": 3.267, "read": 3.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.496, "count": 98, "max_ms": 5.144, "p95_ms": 4.614}, "gstreamer_push": {"avg_ms": 1.264, "count": 98, "max_ms": 1.597, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.352, "count": 98, "max_ms": 87.295, "p95_ms": 79.123}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.3, "count": 98, "max_ms": 109.575, "p95_ms": 100.379}, "yolo11_640_fire_11class": {"avg_ms": 75.596, "count": 98, "max_ms": 90.395, "p95_ms": 86.019}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.643, "count": 98, "max_ms": 97.787, "p95_ms": 95.233}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.4, "count": 98, "max_ms": 122.111, "p95_ms": 111.919}, "read_interval": {"avg_ms": 306.205, "count": 98, "max_ms": 19264.566, "p95_ms": 127.597}, "triton_wait": {"avg_ms": 85.676, "count": 98, "max_ms": 109.589, "p95_ms": 100.392}, "visualize": {"avg_ms": 3.472, "count": 98, "max_ms": 4.364, "p95_ms": 3.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2060}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6045.508, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:36:56+09:00", "last_error_summary": "[h264 @ 0x5555d2890480] reference picture missing during reorder [h264 @ 0x5555d2890480] Missing reference picture, default is 65578 [h264 @ 0x5555d295f300] mmco: unref short failure [h264 @ 0x5555d295f300] illegal short term buffer state detected [h264 @ 0x5555d295f300] error while decoding MB 79 39, bytestream -12 [h264 @ 0x5555d1e45cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5555d1e45cc0] error while decoding MB 0 17, bytestream 7474 [h264 @ 0x5555d1e9ab40] top block ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 116, "h264_decode_error_count": 379, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:36:56+09:00", "last_error_summary": "[h264 @ 0x5555d2890480] reference picture missing during reorder [h264 @ 0x5555d2890480] Missing reference picture, default is 65578 [h264 @ 0x5555d295f300] mmco: unref short failure [h264 @ 0x5555d295f300] illegal short term buffer state detected [h264 @ 0x5555d295f300] error while decoding MB 79 39, bytestream -12 [h264 @ 0x5555d1e45cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5555d1e45cc0] error while decoding MB 0 17, bytestream 7474 [h264 @ 0x5555d1e9ab40] top block ", "last_exit_at": "2026-07-06T21:36:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 358.842, "last_started_at": "2026-07-06T21:30:57+09:00", "pid": 29512, "process": {"exists": false, "pid": 29512}, "restart_count": 125, "restart_count_per_hour": 8, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 126}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.291, "sample_count": 2596, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 247203, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:36:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T21:36:56+09:00", "last_frame_read_age_sec": 0.087, "last_frame_read_at": "2026-07-06T21:36:56+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-06T21:36:56.455025+09:00", "mtime_epoch": 1783341416.4550252, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126522}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:36:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1655.852, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.277, "pushed": 3.277, "read": 3.244}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.499, "count": 98, "max_ms": 5.144, "p95_ms": 4.614}, "gstreamer_push": {"avg_ms": 1.261, "count": 98, "max_ms": 1.597, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.218, "count": 98, "max_ms": 87.295, "p95_ms": 79.123}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.292, "count": 98, "max_ms": 109.575, "p95_ms": 100.379}, "yolo11_640_fire_11class": {"avg_ms": 75.604, "count": 98, "max_ms": 90.395, "p95_ms": 86.019}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.567, "count": 98, "max_ms": 97.787, "p95_ms": 95.233}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.369, "count": 98, "max_ms": 122.111, "p95_ms": 111.919}, "read_interval": {"avg_ms": 308.389, "count": 97, "max_ms": 19264.566, "p95_ms": 127.597}, "triton_wait": {"avg_ms": 85.6, "count": 98, "max_ms": 109.589, "p95_ms": 100.392}, "visualize": {"avg_ms": 3.488, "count": 98, "max_ms": 5.153, "p95_ms": 3.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2060}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6045.469, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 116, "h264_decode_error_count": 379, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:36:56+09:00", "last_error_summary": "[h264 @ 0x5555d2890480] reference picture missing during reorder [h264 @ 0x5555d2890480] Missing reference picture, default is 65578 [h264 @ 0x5555d295f300] mmco: unref short failure [h264 @ 0x5555d295f300] illegal short term buffer state detected [h264 @ 0x5555d295f300] error while decoding MB 79 39, bytestream -12 [h264 @ 0x5555d1e45cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5555d1e45cc0] error while decoding MB 0 17, bytestream 7474 [h264 @ 0x5555d1e9ab40] top block ", "last_exit_at": "2026-07-06T21:36:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 358.849, "last_started_at": "2026-07-06T21:30:57+09:00", "pid": 29512, "process": {"exists": false, "pid": 29512}, "restart_count": 125, "restart_count_per_hour": 8, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 126}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.291, "sample_count": 2596, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 247203, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:36:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T21:36:56+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-06T21:36:56+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-06T21:36:56.455025+09:00", "mtime_epoch": 1783341416.4550252, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126522}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:36:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1655.852, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.276, "pushed": 3.276, "read": 3.243}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.499, "count": 98, "max_ms": 5.144, "p95_ms": 4.614}, "gstreamer_push": {"avg_ms": 1.261, "count": 98, "max_ms": 1.597, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.218, "count": 98, "max_ms": 87.295, "p95_ms": 79.123}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.292, "count": 98, "max_ms": 109.575, "p95_ms": 100.379}, "yolo11_640_fire_11class": {"avg_ms": 75.604, "count": 98, "max_ms": 90.395, "p95_ms": 86.019}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.567, "count": 98, "max_ms": 97.787, "p95_ms": 95.233}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.369, "count": 98, "max_ms": 122.111, "p95_ms": 111.919}, "read_interval": {"avg_ms": 308.389, "count": 97, "max_ms": 19264.566, "p95_ms": 127.597}, "triton_wait": {"avg_ms": 85.6, "count": 98, "max_ms": 109.589, "p95_ms": 100.392}, "visualize": {"avg_ms": 3.488, "count": 98, "max_ms": 5.153, "p95_ms": 3.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2060}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6159.164, "limit_mb": 8796093022207.996}, "detail": {"pid": 29771, "restart_count": 126, "start_count": 127}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 116, "h264_decode_error_count": 379, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:36:56+09:00", "last_error_summary": "[h264 @ 0x5555d2890480] reference picture missing during reorder [h264 @ 0x5555d2890480] Missing reference picture, default is 65578 [h264 @ 0x5555d295f300] mmco: unref short failure [h264 @ 0x5555d295f300] illegal short term buffer state detected [h264 @ 0x5555d295f300] error while decoding MB 79 39, bytestream -12 [h264 @ 0x5555d1e45cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5555d1e45cc0] error while decoding MB 0 17, bytestream 7474 [h264 @ 0x5555d1e9ab40] top block ", "last_exit_at": "2026-07-06T21:36:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:36:58+09:00", "pid": 29771, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.301, "name": "ffmpeg", "pid": 29771, "rss_mb": 139.301, "state": "S (sleeping)", "threads": 106}, "restart_count": 126, "restart_count_per_hour": 9, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 127}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.512, "sample_count": 2558, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 247203, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:36:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.201, "last_frame_pushed_at": "2026-07-06T21:36:56+09:00", "last_frame_read_age_sec": 4.282, "last_frame_read_at": "2026-07-06T21:36:56+09:00", "mjpeg": {"age_sec": 4.183, "exists": true, "mtime": "2026-07-06T21:36:56.547027+09:00", "mtime_epoch": 1783341416.5470266, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 128166}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:37:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1655.852, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.006, "pushed": 2.006, "read": 1.972}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.527, "count": 60, "max_ms": 5.144, "p95_ms": 4.634}, "gstreamer_push": {"avg_ms": 1.285, "count": 60, "max_ms": 1.447, "p95_ms": 1.411}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.109, "count": 60, "max_ms": 87.295, "p95_ms": 79.123}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.774, "count": 60, "max_ms": 109.575, "p95_ms": 96.225}, "yolo11_640_fire_11class": {"avg_ms": 75.77, "count": 60, "max_ms": 87.262, "p95_ms": 83.992}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.828, "count": 60, "max_ms": 96.023, "p95_ms": 95.149}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.713, "count": 60, "max_ms": 122.111, "p95_ms": 109.374}, "read_interval": {"avg_ms": 436.137, "count": 59, "max_ms": 19264.566, "p95_ms": 127.658}, "triton_wait": {"avg_ms": 85.717, "count": 60, "max_ms": 109.589, "p95_ms": 96.237}, "visualize": {"avg_ms": 3.558, "count": 60, "max_ms": 5.153, "p95_ms": 3.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2285}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5989.988, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c29c9bb680] error while decoding MB 115 26, bytestream -21 [in#0/rtsp @ 0x55c29c979e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c29c979e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:37:43+09:00", "last_error_summary": "[h264 @ 0x55c29c9bb680] error while decoding MB 115 26, bytestream -21 [in#0/rtsp @ 0x55c29c979e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c29c979e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:33:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 253.07, "last_started_at": "2026-07-06T21:33:30+09:00", "pid": 29619, "process": {"exists": false, "pid": 29619}, "restart_count": 61, "restart_count_per_hour": 13, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.376, "sample_count": 2096, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 274253, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:37:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T21:37:43+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T21:37:43+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-06T21:37:43.823846+09:00", "mtime_epoch": 1783341463.8238456, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:37:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1543.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.687, "pushed": 0.687, "read": 0.745}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.226, "count": 13, "max_ms": 4.754, "p95_ms": 4.754}, "gstreamer_push": {"avg_ms": 1.392, "count": 12, "max_ms": 1.961, "p95_ms": 1.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.215, "count": 13, "max_ms": 78.276, "p95_ms": 78.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.387, "count": 13, "max_ms": 79.93, "p95_ms": 79.93}, "yolo11_640_fire_11class": {"avg_ms": 57.146, "count": 13, "max_ms": 78.785, "p95_ms": 78.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.074, "count": 13, "max_ms": 90.704, "p95_ms": 90.704}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.935, "count": 12, "max_ms": 103.387, "p95_ms": 103.387}, "read_interval": {"avg_ms": 2708.349, "count": 13, "max_ms": 17813.224, "p95_ms": 17813.224}, "triton_wait": {"avg_ms": 61.795, "count": 13, "max_ms": 90.716, "p95_ms": 90.716}, "visualize": {"avg_ms": 3.693, "count": 13, "max_ms": 4.035, "p95_ms": 4.035}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.457, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2285}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5989.535, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:37:43+09:00", "last_error_summary": "[h264 @ 0x55c29c9bb680] error while decoding MB 115 26, bytestream -21 [in#0/rtsp @ 0x55c29c979e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c29c979e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:37:43+09:00", "last_error_summary": "[h264 @ 0x55c29c9bb680] error while decoding MB 115 26, bytestream -21 [in#0/rtsp @ 0x55c29c979e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c29c979e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:37:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 253.09, "last_started_at": "2026-07-06T21:33:30+09:00", "pid": 29619, "process": {"exists": false, "pid": 29619}, "restart_count": 61, "restart_count_per_hour": 13, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.376, "sample_count": 2097, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 274253, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:37:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T21:37:43+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-06T21:37:43+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-06T21:37:43.823846+09:00", "mtime_epoch": 1783341463.8238456, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:37:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1543.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.226, "count": 13, "max_ms": 4.754, "p95_ms": 4.754}, "gstreamer_push": {"avg_ms": 1.383, "count": 13, "max_ms": 1.961, "p95_ms": 1.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.215, "count": 13, "max_ms": 78.276, "p95_ms": 78.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.387, "count": 13, "max_ms": 79.93, "p95_ms": 79.93}, "yolo11_640_fire_11class": {"avg_ms": 57.146, "count": 13, "max_ms": 78.785, "p95_ms": 78.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.074, "count": 13, "max_ms": 90.704, "p95_ms": 90.704}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.649, "count": 13, "max_ms": 103.387, "p95_ms": 103.387}, "read_interval": {"avg_ms": 2708.349, "count": 13, "max_ms": 17813.224, "p95_ms": 17813.224}, "triton_wait": {"avg_ms": 61.795, "count": 13, "max_ms": 90.716, "p95_ms": 90.716}, "visualize": {"avg_ms": 3.693, "count": 13, "max_ms": 4.035, "p95_ms": 4.035}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.476, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2285}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5989.637, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:37:43+09:00", "last_error_summary": "[h264 @ 0x55c29c9bb680] error while decoding MB 115 26, bytestream -21 [in#0/rtsp @ 0x55c29c979e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c29c979e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:37:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 253.098, "last_started_at": "2026-07-06T21:33:30+09:00", "pid": 29619, "process": {"exists": false, "pid": 29619}, "restart_count": 61, "restart_count_per_hour": 13, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.376, "sample_count": 2097, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 274253, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:37:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T21:37:43+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-06T21:37:43+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-06T21:37:43.823846+09:00", "mtime_epoch": 1783341463.8238456, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:37:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1543.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.226, "count": 13, "max_ms": 4.754, "p95_ms": 4.754}, "gstreamer_push": {"avg_ms": 1.383, "count": 13, "max_ms": 1.961, "p95_ms": 1.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.215, "count": 13, "max_ms": 78.276, "p95_ms": 78.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.387, "count": 13, "max_ms": 79.93, "p95_ms": 79.93}, "yolo11_640_fire_11class": {"avg_ms": 57.146, "count": 13, "max_ms": 78.785, "p95_ms": 78.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.074, "count": 13, "max_ms": 90.704, "p95_ms": 90.704}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.649, "count": 13, "max_ms": 103.387, "p95_ms": 103.387}, "read_interval": {"avg_ms": 2708.349, "count": 13, "max_ms": 17813.224, "p95_ms": 17813.224}, "triton_wait": {"avg_ms": 61.795, "count": 13, "max_ms": 90.716, "p95_ms": 90.716}, "visualize": {"avg_ms": 3.693, "count": 13, "max_ms": 4.035, "p95_ms": 4.035}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.484, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2285}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6166.328, "limit_mb": 8796093022207.996}, "detail": {"pid": 29878, "restart_count": 62, "start_count": 63}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:37:43+09:00", "last_error_summary": "[h264 @ 0x55c29c9bb680] error while decoding MB 115 26, bytestream -21 [in#0/rtsp @ 0x55c29c979e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c29c979e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:37:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:37:46+09:00", "pid": 29878, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.523, "name": "ffmpeg", "pid": 29878, "rss_mb": 206.523, "state": "S (sleeping)", "threads": 106}, "restart_count": 62, "restart_count_per_hour": 14, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.376, "sample_count": 2053, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 274253, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:37:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.602, "last_frame_pushed_at": "2026-07-06T21:37:43+09:00", "last_frame_read_age_sec": 4.676, "last_frame_read_at": "2026-07-06T21:37:43+09:00", "mjpeg": {"age_sec": 4.586, "exists": true, "mtime": "2026-07-06T21:37:43.903847+09:00", "mtime_epoch": 1783341463.9038472, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:37:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1543.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.589, "pushed": 0.589, "read": 0.589}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.226, "count": 13, "max_ms": 4.754, "p95_ms": 4.754}, "gstreamer_push": {"avg_ms": 1.383, "count": 13, "max_ms": 1.961, "p95_ms": 1.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.215, "count": 13, "max_ms": 78.276, "p95_ms": 78.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.387, "count": 13, "max_ms": 79.93, "p95_ms": 79.93}, "yolo11_640_fire_11class": {"avg_ms": 57.146, "count": 13, "max_ms": 78.785, "p95_ms": 78.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.074, "count": 13, "max_ms": 90.704, "p95_ms": 90.704}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.649, "count": 13, "max_ms": 103.387, "p95_ms": 103.387}, "read_interval": {"avg_ms": 2708.349, "count": 13, "max_ms": 17813.224, "p95_ms": 17813.224}, "triton_wait": {"avg_ms": 61.795, "count": 13, "max_ms": 90.716, "p95_ms": 90.716}, "visualize": {"avg_ms": 3.693, "count": 13, "max_ms": 4.035, "p95_ms": 4.035}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.072, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2294}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5997.762, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x557643bbe200] missing picture in access unit with size 18 [h264 @ 0x5576447ceb00] no frame! [in#0/rtsp @ 0x557643bb9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643bb9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557643bbe840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:40:17+09:00", "last_error_summary": "[NULL @ 0x557643bbe200] missing picture in access unit with size 18 [h264 @ 0x5576447ceb00] no frame! [in#0/rtsp @ 0x557643bb9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643bb9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557643bbe840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:37:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.872, "last_started_at": "2026-07-06T21:37:46+09:00", "pid": 29878, "process": {"exists": false, "pid": 29878}, "restart_count": 62, "restart_count_per_hour": 14, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.022, "sample_count": 2100, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 275294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:40:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-06T21:40:17+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-06T21:40:17+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-06T21:40:17.298534+09:00", "mtime_epoch": 1783341617.2985342, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96219}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:40:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1544.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.43, "pushed": 0.43, "read": 0.465}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.102, "count": 12, "max_ms": 4.167, "p95_ms": 4.167}, "gstreamer_push": {"avg_ms": 1.279, "count": 12, "max_ms": 1.406, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.45, "count": 12, "max_ms": 88.146, "p95_ms": 88.146}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.635, "count": 12, "max_ms": 100.354, "p95_ms": 100.354}, "yolo11_640_fire_11class": {"avg_ms": 71.259, "count": 12, "max_ms": 87.249, "p95_ms": 87.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.284, "count": 12, "max_ms": 95.469, "p95_ms": 95.469}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.738, "count": 12, "max_ms": 113.378, "p95_ms": 113.378}, "read_interval": {"avg_ms": 2591.028, "count": 13, "max_ms": 26708.028, "p95_ms": 26708.028}, "triton_wait": {"avg_ms": 79.037, "count": 12, "max_ms": 100.367, "p95_ms": 100.367}, "visualize": {"avg_ms": 3.605, "count": 12, "max_ms": 3.791, "p95_ms": 3.791}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2294}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5997.352, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:40:17+09:00", "last_error_summary": "[NULL @ 0x557643bbe200] missing picture in access unit with size 18 [h264 @ 0x5576447ceb00] no frame! [in#0/rtsp @ 0x557643bb9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643bb9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557643bbe840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:40:17+09:00", "last_error_summary": "[NULL @ 0x557643bbe200] missing picture in access unit with size 18 [h264 @ 0x5576447ceb00] no frame! [in#0/rtsp @ 0x557643bb9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643bb9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557643bbe840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:40:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.908, "last_started_at": "2026-07-06T21:37:46+09:00", "pid": 29878, "process": {"exists": false, "pid": 29878}, "restart_count": 62, "restart_count_per_hour": 14, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.022, "sample_count": 2100, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 275295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:40:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T21:40:17+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-06T21:40:17+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-06T21:40:17.298534+09:00", "mtime_epoch": 1783341617.2985342, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96219}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:40:17+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1544.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.465, "pushed": 0.465, "read": 0.465}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.092, "count": 13, "max_ms": 4.167, "p95_ms": 4.167}, "gstreamer_push": {"avg_ms": 1.28, "count": 13, "max_ms": 1.406, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.876, "count": 13, "max_ms": 88.146, "p95_ms": 88.146}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.17, "count": 13, "max_ms": 100.354, "p95_ms": 100.354}, "yolo11_640_fire_11class": {"avg_ms": 72.096, "count": 13, "max_ms": 87.249, "p95_ms": 87.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.778, "count": 13, "max_ms": 95.469, "p95_ms": 95.469}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.083, "count": 13, "max_ms": 113.378, "p95_ms": 113.378}, "read_interval": {"avg_ms": 2591.028, "count": 13, "max_ms": 26708.028, "p95_ms": 26708.028}, "triton_wait": {"avg_ms": 79.388, "count": 13, "max_ms": 100.367, "p95_ms": 100.367}, "visualize": {"avg_ms": 3.601, "count": 13, "max_ms": 3.791, "p95_ms": 3.791}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.967, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2294}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5997.285, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:40:17+09:00", "last_error_summary": "[NULL @ 0x557643bbe200] missing picture in access unit with size 18 [h264 @ 0x5576447ceb00] no frame! [in#0/rtsp @ 0x557643bb9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643bb9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557643bbe840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:40:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.915, "last_started_at": "2026-07-06T21:37:46+09:00", "pid": 29878, "process": {"exists": false, "pid": 29878}, "restart_count": 62, "restart_count_per_hour": 14, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.022, "sample_count": 2100, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 275295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:40:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T21:40:17+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-06T21:40:17+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T21:40:17.298534+09:00", "mtime_epoch": 1783341617.2985342, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96219}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:40:17+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1544.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.465, "pushed": 0.465, "read": 0.465}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.092, "count": 13, "max_ms": 4.167, "p95_ms": 4.167}, "gstreamer_push": {"avg_ms": 1.28, "count": 13, "max_ms": 1.406, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.876, "count": 13, "max_ms": 88.146, "p95_ms": 88.146}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.17, "count": 13, "max_ms": 100.354, "p95_ms": 100.354}, "yolo11_640_fire_11class": {"avg_ms": 72.096, "count": 13, "max_ms": 87.249, "p95_ms": 87.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.778, "count": 13, "max_ms": 95.469, "p95_ms": 95.469}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.083, "count": 13, "max_ms": 113.378, "p95_ms": 113.378}, "read_interval": {"avg_ms": 2591.028, "count": 13, "max_ms": 26708.028, "p95_ms": 26708.028}, "triton_wait": {"avg_ms": 79.388, "count": 13, "max_ms": 100.367, "p95_ms": 100.367}, "visualize": {"avg_ms": 3.601, "count": 13, "max_ms": 3.791, "p95_ms": 3.791}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.974, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2294}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6123.637, "limit_mb": 8796093022207.996}, "detail": {"pid": 30030, "restart_count": 63, "start_count": 64}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:40:17+09:00", "last_error_summary": "[NULL @ 0x557643bbe200] missing picture in access unit with size 18 [h264 @ 0x5576447ceb00] no frame! [in#0/rtsp @ 0x557643bb9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557643bb9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557643bbe840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:40:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:40:20+09:00", "pid": 30030, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.531, "name": "ffmpeg", "pid": 30030, "rss_mb": 156.367, "state": "S (sleeping)", "threads": 38}, "restart_count": 63, "restart_count_per_hour": 15, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.805, "sample_count": 2059, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 275295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:40:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.728, "last_frame_pushed_at": "2026-07-06T21:40:17+09:00", "last_frame_read_age_sec": 4.827, "last_frame_read_at": "2026-07-06T21:40:17+09:00", "mjpeg": {"age_sec": 4.714, "exists": true, "mtime": "2026-07-06T21:40:17.406536+09:00", "mtime_epoch": 1783341617.406536, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97181}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:40:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1544.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.006, "pushed": 2.006, "read": 2.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.096, "count": 12, "max_ms": 4.167, "p95_ms": 4.167}, "gstreamer_push": {"avg_ms": 1.289, "count": 12, "max_ms": 1.406, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.505, "count": 12, "max_ms": 88.146, "p95_ms": 88.146}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.072, "count": 12, "max_ms": 100.354, "p95_ms": 100.354}, "yolo11_640_fire_11class": {"avg_ms": 73.575, "count": 12, "max_ms": 87.249, "p95_ms": 87.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.562, "count": 12, "max_ms": 95.469, "p95_ms": 95.469}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.11, "count": 12, "max_ms": 113.378, "p95_ms": 113.378}, "read_interval": {"avg_ms": 2321.986, "count": 12, "max_ms": 26708.028, "p95_ms": 26708.028}, "triton_wait": {"avg_ms": 81.388, "count": 12, "max_ms": 100.367, "p95_ms": 100.367}, "visualize": {"avg_ms": 3.605, "count": 12, "max_ms": 3.791, "p95_ms": 3.791}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.983, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5998.172, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5624de2ad880] error while decoding MB 107 40, bytestream -5 [in#0/rtsp @ 0x5624ddc46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624ddc46e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:44:13+09:00", "last_error_summary": "[h264 @ 0x5624de2ad880] error while decoding MB 107 40, bytestream -5 [in#0/rtsp @ 0x5624ddc46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624ddc46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:40:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.501, "last_started_at": "2026-07-06T21:40:20+09:00", "pid": 30030, "process": {"exists": false, "pid": 30030}, "restart_count": 63, "restart_count_per_hour": 15, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.196, "sample_count": 2052, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277066, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:44:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.001, "last_frame_pushed_at": "2026-07-06T21:44:13+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T21:44:13+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-06T21:44:13.562740+09:00", "mtime_epoch": 1783341853.5627398, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:44:13+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.031, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.797, "pushed": 0.797, "read": 0.797}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.155, "count": 14, "max_ms": 5.0, "p95_ms": 5.0}, "gstreamer_push": {"avg_ms": 1.297, "count": 14, "max_ms": 1.581, "p95_ms": 1.581}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.399, "count": 14, "max_ms": 76.72, "p95_ms": 76.72}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.251, "count": 14, "max_ms": 99.583, "p95_ms": 99.583}, "yolo11_640_fire_11class": {"avg_ms": 62.647, "count": 14, "max_ms": 85.984, "p95_ms": 85.984}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.252, "count": 14, "max_ms": 90.396, "p95_ms": 90.396}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.834, "count": 14, "max_ms": 112.26, "p95_ms": 112.26}, "read_interval": {"avg_ms": 2484.188, "count": 14, "max_ms": 17289.346, "p95_ms": 17289.346}, "triton_wait": {"avg_ms": 66.053, "count": 14, "max_ms": 99.595, "p95_ms": 99.595}, "visualize": {"avg_ms": 3.656, "count": 14, "max_ms": 4.293, "p95_ms": 4.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.555, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5998.043, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:44:13+09:00", "last_error_summary": "[h264 @ 0x5624de2ad880] error while decoding MB 107 40, bytestream -5 [in#0/rtsp @ 0x5624ddc46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624ddc46e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:44:13+09:00", "last_error_summary": "[h264 @ 0x5624de2ad880] error while decoding MB 107 40, bytestream -5 [in#0/rtsp @ 0x5624ddc46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624ddc46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:44:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.515, "last_started_at": "2026-07-06T21:40:20+09:00", "pid": 30030, "process": {"exists": false, "pid": 30030}, "restart_count": 63, "restart_count_per_hour": 15, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.196, "sample_count": 2052, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277066, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:44:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T21:44:13+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-06T21:44:13+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-06T21:44:13.562740+09:00", "mtime_epoch": 1783341853.5627398, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:44:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.031, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.797, "pushed": 0.797, "read": 0.797}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.155, "count": 14, "max_ms": 5.0, "p95_ms": 5.0}, "gstreamer_push": {"avg_ms": 1.297, "count": 14, "max_ms": 1.581, "p95_ms": 1.581}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.399, "count": 14, "max_ms": 76.72, "p95_ms": 76.72}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.251, "count": 14, "max_ms": 99.583, "p95_ms": 99.583}, "yolo11_640_fire_11class": {"avg_ms": 62.647, "count": 14, "max_ms": 85.984, "p95_ms": 85.984}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.252, "count": 14, "max_ms": 90.396, "p95_ms": 90.396}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.834, "count": 14, "max_ms": 112.26, "p95_ms": 112.26}, "read_interval": {"avg_ms": 2484.188, "count": 14, "max_ms": 17289.346, "p95_ms": 17289.346}, "triton_wait": {"avg_ms": 66.053, "count": 14, "max_ms": 99.595, "p95_ms": 99.595}, "visualize": {"avg_ms": 3.656, "count": 14, "max_ms": 4.293, "p95_ms": 4.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.566, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5997.984, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:44:13+09:00", "last_error_summary": "[h264 @ 0x5624de2ad880] error while decoding MB 107 40, bytestream -5 [in#0/rtsp @ 0x5624ddc46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624ddc46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:44:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.522, "last_started_at": "2026-07-06T21:40:20+09:00", "pid": 30030, "process": {"exists": false, "pid": 30030}, "restart_count": 63, "restart_count_per_hour": 15, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.196, "sample_count": 2052, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277066, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:44:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-06T21:44:13+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-06T21:44:13+09:00", "mjpeg": {"age_sec": 0.011, "exists": true, "mtime": "2026-07-06T21:44:13.630741+09:00", "mtime_epoch": 1783341853.6307411, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:44:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.031, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.797, "pushed": 0.797, "read": 0.797}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.155, "count": 14, "max_ms": 5.0, "p95_ms": 5.0}, "gstreamer_push": {"avg_ms": 1.297, "count": 14, "max_ms": 1.581, "p95_ms": 1.581}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.399, "count": 14, "max_ms": 76.72, "p95_ms": 76.72}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.251, "count": 14, "max_ms": 99.583, "p95_ms": 99.583}, "yolo11_640_fire_11class": {"avg_ms": 62.647, "count": 14, "max_ms": 85.984, "p95_ms": 85.984}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.252, "count": 14, "max_ms": 90.396, "p95_ms": 90.396}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.834, "count": 14, "max_ms": 112.26, "p95_ms": 112.26}, "read_interval": {"avg_ms": 2484.188, "count": 14, "max_ms": 17289.346, "p95_ms": 17289.346}, "triton_wait": {"avg_ms": 66.053, "count": 14, "max_ms": 99.595, "p95_ms": 99.595}, "visualize": {"avg_ms": 3.656, "count": 14, "max_ms": 4.293, "p95_ms": 4.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.573, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6069.156, "limit_mb": 8796093022207.996}, "detail": {"pid": 30182, "restart_count": 64, "start_count": 65}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:44:13+09:00", "last_error_summary": "[h264 @ 0x5624de2ad880] error while decoding MB 107 40, bytestream -5 [in#0/rtsp @ 0x5624ddc46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624ddc46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:44:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:44:15+09:00", "pid": 30182, "process": {"exists": true, "fd_count": 5, "hwm_mb": 96.691, "name": "ffmpeg", "pid": 30182, "rss_mb": 96.691, "state": "S (sleeping)", "threads": 19}, "restart_count": 64, "restart_count_per_hour": 16, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.212, "sample_count": 2014, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277066, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:44:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.322, "last_frame_pushed_at": "2026-07-06T21:44:13+09:00", "last_frame_read_age_sec": 4.384, "last_frame_read_at": "2026-07-06T21:44:13+09:00", "mjpeg": {"age_sec": 4.311, "exists": true, "mtime": "2026-07-06T21:44:13.630741+09:00", "mtime_epoch": 1783341853.6307411, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:44:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.031, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.64, "pushed": 0.64, "read": 0.64}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.155, "count": 14, "max_ms": 5.0, "p95_ms": 5.0}, "gstreamer_push": {"avg_ms": 1.297, "count": 14, "max_ms": 1.581, "p95_ms": 1.581}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.399, "count": 14, "max_ms": 76.72, "p95_ms": 76.72}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.251, "count": 14, "max_ms": 99.583, "p95_ms": 99.583}, "yolo11_640_fire_11class": {"avg_ms": 62.647, "count": 14, "max_ms": 85.984, "p95_ms": 85.984}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.252, "count": 14, "max_ms": 90.396, "p95_ms": 90.396}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.834, "count": 14, "max_ms": 112.26, "p95_ms": 112.26}, "read_interval": {"avg_ms": 2484.188, "count": 14, "max_ms": 17289.346, "p95_ms": 17289.346}, "triton_wait": {"avg_ms": 66.053, "count": 14, "max_ms": 99.595, "p95_ms": 99.595}, "visualize": {"avg_ms": 3.656, "count": 14, "max_ms": 4.293, "p95_ms": 4.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.874, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2316}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6313.031, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:44:13+09:00", "last_error_summary": "[h264 @ 0x5624de2ad880] error while decoding MB 107 40, bytestream -5 [in#0/rtsp @ 0x5624ddc46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624ddc46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:44:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.248, "last_started_at": "2026-07-06T21:44:15+09:00", "pid": 30182, "process": {"exists": true, "fd_count": 5, "hwm_mb": 329.508, "name": "ffmpeg", "pid": 30182, "rss_mb": 329.508, "state": "S (sleeping)", "threads": 106}, "restart_count": 64, "restart_count_per_hour": 16, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.968, "sample_count": 2082, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 278014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:46:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.056, "last_frame_pushed_at": "2026-07-06T21:46:03+09:00", "last_frame_read_age_sec": 30.153, "last_frame_read_at": "2026-07-06T21:46:03+09:00", "mjpeg": {"age_sec": 30.04, "exists": true, "mtime": "2026-07-06T21:46:03.148710+09:00", "mtime_epoch": 1783341963.1487105, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:46:33+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.547, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2316}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6304.652, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:44:13+09:00", "last_error_summary": "[h264 @ 0x5624de2ad880] error while decoding MB 107 40, bytestream -5 [in#0/rtsp @ 0x5624ddc46e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624ddc46e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:44:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.26, "last_started_at": "2026-07-06T21:44:15+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 64, "restart_count_per_hour": 16, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.968, "sample_count": 2082, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 278014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:46:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.068, "last_frame_pushed_at": "2026-07-06T21:46:03+09:00", "last_frame_read_age_sec": 30.166, "last_frame_read_at": "2026-07-06T21:46:03+09:00", "mjpeg": {"age_sec": 30.053, "exists": true, "mtime": "2026-07-06T21:46:03.148710+09:00", "mtime_epoch": 1783341963.1487105, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:46:33+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.547, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2316}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6011.383, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556a7c933a80] error while decoding MB 78 30, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:46:33+09:00", "last_error_summary": "[h264 @ 0x556a7c933a80] error while decoding MB 78 30, bytestream -5", "last_exit_at": "2026-07-06T21:44:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.311, "last_started_at": "2026-07-06T21:44:15+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 64, "restart_count_per_hour": 16, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.968, "sample_count": 2082, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 278014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:46:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.119, "last_frame_pushed_at": "2026-07-06T21:46:03+09:00", "last_frame_read_age_sec": 30.216, "last_frame_read_at": "2026-07-06T21:46:03+09:00", "mjpeg": {"age_sec": 30.104, "exists": true, "mtime": "2026-07-06T21:46:03.148710+09:00", "mtime_epoch": 1783341963.1487105, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:46:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.547, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2316}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6029.363, "limit_mb": 8796093022207.996}, "detail": {"pid": 30334, "restart_count": 65, "start_count": 66}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:46:33+09:00", "last_error_summary": "[h264 @ 0x556a7c933a80] error while decoding MB 78 30, bytestream -5", "last_exit_at": "2026-07-06T21:44:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:46:35+09:00", "pid": 30334, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.895, "name": "ffmpeg", "pid": 30334, "rss_mb": 57.895, "state": "S (sleeping)", "threads": 1}, "restart_count": 65, "restart_count_per_hour": 17, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.294, "sample_count": 2042, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 278014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:46:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.567, "last_frame_pushed_at": "2026-07-06T21:46:03+09:00", "last_frame_read_age_sec": 34.664, "last_frame_read_at": "2026-07-06T21:46:03+09:00", "mjpeg": {"age_sec": 34.552, "exists": true, "mtime": "2026-07-06T21:46:03.148710+09:00", "mtime_epoch": 1783341963.1487105, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:46:37+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.547, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2324}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6015.438, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556ac99c3940] error while decoding MB 78 22, bytestream -9 [in#0/rtsp @ 0x556ac90efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556ac90efe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:48:47+09:00", "last_error_summary": "[h264 @ 0x556ac99c3940] error while decoding MB 78 22, bytestream -9 [in#0/rtsp @ 0x556ac90efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556ac90efe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:44:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.326, "last_started_at": "2026-07-06T21:46:35+09:00", "pid": 30334, "process": {"exists": false, "pid": 30334}, "restart_count": 65, "restart_count_per_hour": 17, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.095, "sample_count": 1830, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 278882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-06T21:48:46+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T21:48:46+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T21:48:46.967673+09:00", "mtime_epoch": 1783342126.967673, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:48:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.805, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.691, "pushed": 0.691, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.124, "count": 12, "max_ms": 4.918, "p95_ms": 4.918}, "gstreamer_push": {"avg_ms": 1.269, "count": 12, "max_ms": 1.355, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.413, "count": 12, "max_ms": 79.19, "p95_ms": 79.19}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.964, "count": 12, "max_ms": 93.808, "p95_ms": 93.808}, "yolo11_640_fire_11class": {"avg_ms": 73.44, "count": 12, "max_ms": 87.086, "p95_ms": 87.086}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.063, "count": 12, "max_ms": 87.735, "p95_ms": 87.735}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.299, "count": 12, "max_ms": 103.737, "p95_ms": 103.737}, "read_interval": {"avg_ms": 2570.917, "count": 13, "max_ms": 16129.608, "p95_ms": 16129.608}, "triton_wait": {"avg_ms": 81.54, "count": 12, "max_ms": 93.819, "p95_ms": 93.819}, "visualize": {"avg_ms": 3.53, "count": 12, "max_ms": 3.803, "p95_ms": 3.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.354, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2324}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6014.555, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:48:47+09:00", "last_error_summary": "[h264 @ 0x556ac99c3940] error while decoding MB 78 22, bytestream -9 [in#0/rtsp @ 0x556ac90efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556ac90efe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:48:47+09:00", "last_error_summary": "[h264 @ 0x556ac99c3940] error while decoding MB 78 22, bytestream -9 [in#0/rtsp @ 0x556ac90efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556ac90efe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:48:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.367, "last_started_at": "2026-07-06T21:46:35+09:00", "pid": 30334, "process": {"exists": false, "pid": 30334}, "restart_count": 65, "restart_count_per_hour": 17, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.073, "sample_count": 1831, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 278883, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T21:48:47+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-06T21:48:46+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-06T21:48:46.967673+09:00", "mtime_epoch": 1783342126.967673, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:48:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.747, "pushed": 0.747, "read": 0.747}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.12, "count": 13, "max_ms": 4.918, "p95_ms": 4.918}, "gstreamer_push": {"avg_ms": 1.268, "count": 13, "max_ms": 1.355, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.746, "count": 13, "max_ms": 79.19, "p95_ms": 79.19}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.552, "count": 13, "max_ms": 93.808, "p95_ms": 93.808}, "yolo11_640_fire_11class": {"avg_ms": 73.32, "count": 13, "max_ms": 87.086, "p95_ms": 87.086}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.361, "count": 13, "max_ms": 87.735, "p95_ms": 87.735}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.245, "count": 13, "max_ms": 103.737, "p95_ms": 103.737}, "read_interval": {"avg_ms": 2570.917, "count": 13, "max_ms": 16129.608, "p95_ms": 16129.608}, "triton_wait": {"avg_ms": 81.494, "count": 13, "max_ms": 93.819, "p95_ms": 93.819}, "visualize": {"avg_ms": 3.529, "count": 13, "max_ms": 3.803, "p95_ms": 3.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.395, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2324}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6014.605, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:48:47+09:00", "last_error_summary": "[h264 @ 0x556ac99c3940] error while decoding MB 78 22, bytestream -9 [in#0/rtsp @ 0x556ac90efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556ac90efe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:48:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.376, "last_started_at": "2026-07-06T21:46:35+09:00", "pid": 30334, "process": {"exists": false, "pid": 30334}, "restart_count": 65, "restart_count_per_hour": 17, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.073, "sample_count": 1831, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 278883, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T21:48:47+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T21:48:46+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-06T21:48:46.967673+09:00", "mtime_epoch": 1783342126.967673, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:48:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.747, "pushed": 0.747, "read": 0.747}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.12, "count": 13, "max_ms": 4.918, "p95_ms": 4.918}, "gstreamer_push": {"avg_ms": 1.268, "count": 13, "max_ms": 1.355, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.746, "count": 13, "max_ms": 79.19, "p95_ms": 79.19}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.552, "count": 13, "max_ms": 93.808, "p95_ms": 93.808}, "yolo11_640_fire_11class": {"avg_ms": 73.32, "count": 13, "max_ms": 87.086, "p95_ms": 87.086}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.361, "count": 13, "max_ms": 87.735, "p95_ms": 87.735}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.245, "count": 13, "max_ms": 103.737, "p95_ms": 103.737}, "read_interval": {"avg_ms": 2570.917, "count": 13, "max_ms": 16129.608, "p95_ms": 16129.608}, "triton_wait": {"avg_ms": 81.494, "count": 13, "max_ms": 93.819, "p95_ms": 93.819}, "visualize": {"avg_ms": 3.529, "count": 13, "max_ms": 3.803, "p95_ms": 3.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.404, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2324}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6192.594, "limit_mb": 8796093022207.996}, "detail": {"pid": 30486, "restart_count": 66, "start_count": 67}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:48:47+09:00", "last_error_summary": "[h264 @ 0x556ac99c3940] error while decoding MB 78 22, bytestream -9 [in#0/rtsp @ 0x556ac90efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556ac90efe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:48:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:48:49+09:00", "pid": 30486, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.195, "name": "ffmpeg", "pid": 30486, "rss_mb": 204.195, "state": "S (sleeping)", "threads": 106}, "restart_count": 66, "restart_count_per_hour": 18, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.073, "sample_count": 1831, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 278883, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.289, "last_frame_pushed_at": "2026-07-06T21:48:47+09:00", "last_frame_read_age_sec": 4.384, "last_frame_read_at": "2026-07-06T21:48:46+09:00", "mjpeg": {"age_sec": 4.268, "exists": true, "mtime": "2026-07-06T21:48:47.079675+09:00", "mtime_epoch": 1783342127.0796752, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103160}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:48:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1545.805, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.6, "pushed": 0.6, "read": 0.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.12, "count": 13, "max_ms": 4.918, "p95_ms": 4.918}, "gstreamer_push": {"avg_ms": 1.268, "count": 13, "max_ms": 1.355, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.746, "count": 13, "max_ms": 79.19, "p95_ms": 79.19}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.552, "count": 13, "max_ms": 93.808, "p95_ms": 93.808}, "yolo11_640_fire_11class": {"avg_ms": 73.32, "count": 13, "max_ms": 87.086, "p95_ms": 87.086}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.361, "count": 13, "max_ms": 87.735, "p95_ms": 87.735}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.245, "count": 13, "max_ms": 103.737, "p95_ms": 103.737}, "read_interval": {"avg_ms": 2570.917, "count": 13, "max_ms": 16129.608, "p95_ms": 16129.608}, "triton_wait": {"avg_ms": 81.494, "count": 13, "max_ms": 93.819, "p95_ms": 93.819}, "visualize": {"avg_ms": 3.529, "count": 13, "max_ms": 3.803, "p95_ms": 3.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.677, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2332}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6035.703, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557853ae08c0] error while decoding MB 111 15, bytestream -7 [in#0/rtsp @ 0x55785347ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55785347ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:51:12+09:00", "last_error_summary": "[h264 @ 0x557853ae08c0] error while decoding MB 111 15, bytestream -7 [in#0/rtsp @ 0x55785347ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55785347ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:48:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.07, "last_started_at": "2026-07-06T21:48:49+09:00", "pid": 30486, "process": {"exists": false, "pid": 30486}, "restart_count": 66, "restart_count_per_hour": 17, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.127, "sample_count": 1853, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 279867, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:51:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.264, "last_frame_pushed_at": "2026-07-06T21:51:12+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-06T21:51:12+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-06T21:51:12.170312+09:00", "mtime_epoch": 1783342272.170312, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101362}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:51:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1546.32, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.742, "pushed": 0.742, "read": 0.799}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.226, "count": 13, "max_ms": 4.667, "p95_ms": 4.667}, "gstreamer_push": {"avg_ms": 1.283, "count": 13, "max_ms": 1.386, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.886, "count": 13, "max_ms": 59.046, "p95_ms": 59.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.786, "count": 13, "max_ms": 66.655, "p95_ms": 66.655}, "yolo11_640_fire_11class": {"avg_ms": 54.68, "count": 13, "max_ms": 62.36, "p95_ms": 62.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.185, "count": 13, "max_ms": 60.885, "p95_ms": 60.885}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.974, "count": 13, "max_ms": 79.782, "p95_ms": 79.782}, "read_interval": {"avg_ms": 2470.81, "count": 14, "max_ms": 17133.345, "p95_ms": 17133.345}, "triton_wait": {"avg_ms": 56.844, "count": 13, "max_ms": 66.668, "p95_ms": 66.668}, "visualize": {"avg_ms": 3.799, "count": 13, "max_ms": 5.397, "p95_ms": 5.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.518, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2332}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6035.18, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:51:12+09:00", "last_error_summary": "[h264 @ 0x557853ae08c0] error while decoding MB 111 15, bytestream -7 [in#0/rtsp @ 0x55785347ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55785347ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:51:12+09:00", "last_error_summary": "[h264 @ 0x557853ae08c0] error while decoding MB 111 15, bytestream -7 [in#0/rtsp @ 0x55785347ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55785347ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:51:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.095, "last_started_at": "2026-07-06T21:48:49+09:00", "pid": 30486, "process": {"exists": false, "pid": 30486}, "restart_count": 66, "restart_count_per_hour": 17, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.127, "sample_count": 1854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 279868, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:51:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-06T21:51:12+09:00", "last_frame_read_age_sec": 0.083, "last_frame_read_at": "2026-07-06T21:51:12+09:00", "mjpeg": {"age_sec": 0.271, "exists": true, "mtime": "2026-07-06T21:51:12.170312+09:00", "mtime_epoch": 1783342272.170312, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101362}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:51:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1546.32, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.798, "pushed": 0.798, "read": 0.798}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.281, "count": 14, "max_ms": 5.005, "p95_ms": 5.005}, "gstreamer_push": {"avg_ms": 1.274, "count": 14, "max_ms": 1.386, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.773, "count": 14, "max_ms": 59.046, "p95_ms": 59.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.615, "count": 14, "max_ms": 66.655, "p95_ms": 66.655}, "yolo11_640_fire_11class": {"avg_ms": 54.545, "count": 14, "max_ms": 62.36, "p95_ms": 62.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.089, "count": 14, "max_ms": 60.885, "p95_ms": 60.885}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.107, "count": 14, "max_ms": 79.782, "p95_ms": 79.782}, "read_interval": {"avg_ms": 2470.81, "count": 14, "max_ms": 17133.345, "p95_ms": 17133.345}, "triton_wait": {"avg_ms": 56.63, "count": 14, "max_ms": 66.668, "p95_ms": 66.668}, "visualize": {"avg_ms": 4.003, "count": 14, "max_ms": 6.657, "p95_ms": 6.657}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.541, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2332}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6035.199, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:51:12+09:00", "last_error_summary": "[h264 @ 0x557853ae08c0] error while decoding MB 111 15, bytestream -7 [in#0/rtsp @ 0x55785347ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55785347ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:51:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.102, "last_started_at": "2026-07-06T21:48:49+09:00", "pid": 30486, "process": {"exists": false, "pid": 30486}, "restart_count": 66, "restart_count_per_hour": 17, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.127, "sample_count": 1854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 279868, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:51:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T21:51:12+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-06T21:51:12+09:00", "mjpeg": {"age_sec": 0.278, "exists": true, "mtime": "2026-07-06T21:51:12.170312+09:00", "mtime_epoch": 1783342272.170312, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101362}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:51:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1546.32, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.798, "pushed": 0.798, "read": 0.798}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.281, "count": 14, "max_ms": 5.005, "p95_ms": 5.005}, "gstreamer_push": {"avg_ms": 1.274, "count": 14, "max_ms": 1.386, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.773, "count": 14, "max_ms": 59.046, "p95_ms": 59.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.615, "count": 14, "max_ms": 66.655, "p95_ms": 66.655}, "yolo11_640_fire_11class": {"avg_ms": 54.545, "count": 14, "max_ms": 62.36, "p95_ms": 62.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.089, "count": 14, "max_ms": 60.885, "p95_ms": 60.885}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.107, "count": 14, "max_ms": 79.782, "p95_ms": 79.782}, "read_interval": {"avg_ms": 2470.81, "count": 14, "max_ms": 17133.345, "p95_ms": 17133.345}, "triton_wait": {"avg_ms": 56.63, "count": 14, "max_ms": 66.668, "p95_ms": 66.668}, "visualize": {"avg_ms": 4.003, "count": 14, "max_ms": 6.657, "p95_ms": 6.657}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.548, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2332}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6206.547, "limit_mb": 8796093022207.996}, "detail": {"pid": 30638, "restart_count": 67, "start_count": 68}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:51:12+09:00", "last_error_summary": "[h264 @ 0x557853ae08c0] error while decoding MB 111 15, bytestream -7 [in#0/rtsp @ 0x55785347ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55785347ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:51:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:51:15+09:00", "pid": 30638, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.824, "name": "ffmpeg", "pid": 30638, "rss_mb": 206.824, "state": "S (sleeping)", "threads": 106}, "restart_count": 67, "restart_count_per_hour": 18, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.127, "sample_count": 1854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 279868, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:51:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.881, "last_frame_pushed_at": "2026-07-06T21:51:12+09:00", "last_frame_read_age_sec": 4.955, "last_frame_read_at": "2026-07-06T21:51:12+09:00", "mjpeg": {"age_sec": 4.867, "exists": true, "mtime": "2026-07-06T21:51:12.446317+09:00", "mtime_epoch": 1783342272.446317, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100114}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:51:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1546.32, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.625, "pushed": 0.625, "read": 0.625}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.281, "count": 14, "max_ms": 5.005, "p95_ms": 5.005}, "gstreamer_push": {"avg_ms": 1.274, "count": 14, "max_ms": 1.386, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.773, "count": 14, "max_ms": 59.046, "p95_ms": 59.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.615, "count": 14, "max_ms": 66.655, "p95_ms": 66.655}, "yolo11_640_fire_11class": {"avg_ms": 54.545, "count": 14, "max_ms": 62.36, "p95_ms": 62.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.089, "count": 14, "max_ms": 60.885, "p95_ms": 60.885}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.107, "count": 14, "max_ms": 79.782, "p95_ms": 79.782}, "read_interval": {"avg_ms": 2470.81, "count": 14, "max_ms": 17133.345, "p95_ms": 17133.345}, "triton_wait": {"avg_ms": 56.63, "count": 14, "max_ms": 66.668, "p95_ms": 66.668}, "visualize": {"avg_ms": 4.003, "count": 14, "max_ms": 6.657, "p95_ms": 6.657}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.413, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6058.434, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a6d21da200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a6d21da200] error while decoding MB 0 29, bytestream 5842 [h264 @ 0x55a6d2f7ce40] error while decoding MB 26 27, bytestream -5 [in#0/rtsp @ 0x55a6d21bee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6d21bee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 117, "h264_decode_error_count": 381, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:51:29+09:00", "last_error_summary": "[h264 @ 0x55a6d21da200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a6d21da200] error while decoding MB 0 29, bytestream 5842 [h264 @ 0x55a6d2f7ce40] error while decoding MB 26 27, bytestream -5 [in#0/rtsp @ 0x55a6d21bee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6d21bee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:36:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 871.028, "last_started_at": "2026-07-06T21:36:58+09:00", "pid": 29771, "process": {"exists": false, "pid": 29771}, "restart_count": 126, "restart_count_per_hour": 7, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 127}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.925, "sample_count": 2670, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255303, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:51:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T21:51:29+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T21:51:29+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-06T21:51:29.698631+09:00", "mtime_epoch": 1783342289.698631, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:51:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1659.617, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 114, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.802, "pushed": 3.802, "read": 3.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.624, "count": 114, "max_ms": 4.459, "p95_ms": 4.234}, "gstreamer_push": {"avg_ms": 1.302, "count": 114, "max_ms": 1.512, "p95_ms": 1.441}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.088, "count": 114, "max_ms": 79.891, "p95_ms": 67.293}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.995, "count": 114, "max_ms": 94.095, "p95_ms": 85.297}, "yolo11_640_fire_11class": {"avg_ms": 59.834, "count": 114, "max_ms": 80.436, "p95_ms": 75.589}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.176, "count": 114, "max_ms": 89.399, "p95_ms": 84.603}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.214, "count": 114, "max_ms": 106.539, "p95_ms": 98.63}, "read_interval": {"avg_ms": 263.022, "count": 114, "max_ms": 19367.622, "p95_ms": 112.889}, "triton_wait": {"avg_ms": 67.883, "count": 114, "max_ms": 94.107, "p95_ms": 86.497}, "visualize": {"avg_ms": 3.687, "count": 114, "max_ms": 4.251, "p95_ms": 4.1}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6057.66, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:51:29+09:00", "last_error_summary": "[h264 @ 0x55a6d21da200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a6d21da200] error while decoding MB 0 29, bytestream 5842 [h264 @ 0x55a6d2f7ce40] error while decoding MB 26 27, bytestream -5 [in#0/rtsp @ 0x55a6d21bee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6d21bee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 117, "h264_decode_error_count": 381, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:51:29+09:00", "last_error_summary": "[h264 @ 0x55a6d21da200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a6d21da200] error while decoding MB 0 29, bytestream 5842 [h264 @ 0x55a6d2f7ce40] error while decoding MB 26 27, bytestream -5 [in#0/rtsp @ 0x55a6d21bee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6d21bee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:51:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 871.073, "last_started_at": "2026-07-06T21:36:58+09:00", "pid": 29771, "process": {"exists": false, "pid": 29771}, "restart_count": 126, "restart_count_per_hour": 7, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 127}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.828, "sample_count": 2671, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:51:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-06T21:51:29+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T21:51:29+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-06T21:51:29.698631+09:00", "mtime_epoch": 1783342289.698631, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:51:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1659.617, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.811, "pushed": 3.811, "read": 3.777}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.616, "count": 114, "max_ms": 4.459, "p95_ms": 4.234}, "gstreamer_push": {"avg_ms": 1.301, "count": 114, "max_ms": 1.512, "p95_ms": 1.441}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.326, "count": 114, "max_ms": 79.891, "p95_ms": 73.736}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.201, "count": 114, "max_ms": 94.095, "p95_ms": 85.297}, "yolo11_640_fire_11class": {"avg_ms": 60.027, "count": 114, "max_ms": 80.436, "p95_ms": 75.98}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.445, "count": 114, "max_ms": 89.399, "p95_ms": 85.248}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.469, "count": 114, "max_ms": 106.539, "p95_ms": 98.63}, "read_interval": {"avg_ms": 264.638, "count": 113, "max_ms": 19367.622, "p95_ms": 112.889}, "triton_wait": {"avg_ms": 68.147, "count": 114, "max_ms": 94.107, "p95_ms": 86.497}, "visualize": {"avg_ms": 3.686, "count": 114, "max_ms": 4.251, "p95_ms": 4.1}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6058.824, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 117, "h264_decode_error_count": 381, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:51:29+09:00", "last_error_summary": "[h264 @ 0x55a6d21da200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a6d21da200] error while decoding MB 0 29, bytestream 5842 [h264 @ 0x55a6d2f7ce40] error while decoding MB 26 27, bytestream -5 [in#0/rtsp @ 0x55a6d21bee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6d21bee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:51:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 871.08, "last_started_at": "2026-07-06T21:36:58+09:00", "pid": 29771, "process": {"exists": false, "pid": 29771}, "restart_count": 126, "restart_count_per_hour": 7, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 127}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.925, "sample_count": 2670, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:51:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T21:51:29+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T21:51:29+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T21:51:29.698631+09:00", "mtime_epoch": 1783342289.698631, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:51:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1659.617, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.81, "pushed": 3.81, "read": 3.776}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.616, "count": 114, "max_ms": 4.459, "p95_ms": 4.234}, "gstreamer_push": {"avg_ms": 1.301, "count": 114, "max_ms": 1.512, "p95_ms": 1.441}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.326, "count": 114, "max_ms": 79.891, "p95_ms": 73.736}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.201, "count": 114, "max_ms": 94.095, "p95_ms": 85.297}, "yolo11_640_fire_11class": {"avg_ms": 60.027, "count": 114, "max_ms": 80.436, "p95_ms": 75.98}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.445, "count": 114, "max_ms": 89.399, "p95_ms": 85.248}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.469, "count": 114, "max_ms": 106.539, "p95_ms": 98.63}, "read_interval": {"avg_ms": 264.638, "count": 113, "max_ms": 19367.622, "p95_ms": 112.889}, "triton_wait": {"avg_ms": 68.147, "count": 114, "max_ms": 94.107, "p95_ms": 86.497}, "visualize": {"avg_ms": 3.686, "count": 114, "max_ms": 4.251, "p95_ms": 4.1}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6180.238, "limit_mb": 8796093022207.996}, "detail": {"pid": 30790, "restart_count": 127, "start_count": 128}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 117, "h264_decode_error_count": 381, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:51:29+09:00", "last_error_summary": "[h264 @ 0x55a6d21da200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a6d21da200] error while decoding MB 0 29, bytestream 5842 [h264 @ 0x55a6d2f7ce40] error while decoding MB 26 27, bytestream -5 [in#0/rtsp @ 0x55a6d21bee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a6d21bee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:51:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:51:32+09:00", "pid": 30790, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.914, "name": "ffmpeg", "pid": 30790, "rss_mb": 149.914, "state": "S (sleeping)", "threads": 106}, "restart_count": 127, "restart_count_per_hour": 8, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 128}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.039, "sample_count": 2617, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:51:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.397, "last_frame_pushed_at": "2026-07-06T21:51:29+09:00", "last_frame_read_age_sec": 4.496, "last_frame_read_at": "2026-07-06T21:51:29+09:00", "mjpeg": {"age_sec": 4.378, "exists": true, "mtime": "2026-07-06T21:51:29.810633+09:00", "mtime_epoch": 1783342289.8106332, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113881}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:51:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1659.617, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.234, "pushed": 2.234, "read": 2.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.575, "count": 67, "max_ms": 4.41, "p95_ms": 4.13}, "gstreamer_push": {"avg_ms": 1.305, "count": 67, "max_ms": 1.512, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.627, "count": 67, "max_ms": 79.891, "p95_ms": 74.976}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.434, "count": 67, "max_ms": 94.095, "p95_ms": 83.749}, "yolo11_640_fire_11class": {"avg_ms": 61.06, "count": 67, "max_ms": 80.436, "p95_ms": 78.697}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.028, "count": 67, "max_ms": 89.399, "p95_ms": 87.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.097, "count": 67, "max_ms": 106.539, "p95_ms": 100.986}, "read_interval": {"avg_ms": 387.752, "count": 66, "max_ms": 19367.622, "p95_ms": 113.115}, "triton_wait": {"avg_ms": 69.803, "count": 67, "max_ms": 94.107, "p95_ms": 89.131}, "visualize": {"avg_ms": 3.676, "count": 67, "max_ms": 4.149, "p95_ms": 4.096}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2344}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6001.301, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558f8491a180] error while decoding MB 0 28, bytestream -9 [in#0/rtsp @ 0x558f84892e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558f84892e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:54:31+09:00", "last_error_summary": "[h264 @ 0x558f8491a180] error while decoding MB 0 28, bytestream -9 [in#0/rtsp @ 0x558f84892e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558f84892e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:51:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.402, "last_started_at": "2026-07-06T21:51:15+09:00", "pid": 30638, "process": {"exists": false, "pid": 30638}, "restart_count": 67, "restart_count_per_hour": 17, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.23, "sample_count": 2132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281387, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:54:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-06T21:54:31+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T21:54:31+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T21:54:31.657951+09:00", "mtime_epoch": 1783342471.6579514, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106430}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:54:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.094, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 23, "pushed_frames": 23, "read_frames": 23, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.769, "pushed": 0.769, "read": 0.769}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.341, "count": 23, "max_ms": 5.592, "p95_ms": 5.568}, "gstreamer_push": {"avg_ms": 1.314, "count": 23, "max_ms": 2.155, "p95_ms": 1.482}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.704, "count": 23, "max_ms": 81.252, "p95_ms": 80.292}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.687, "count": 23, "max_ms": 98.225, "p95_ms": 94.535}, "yolo11_640_fire_11class": {"avg_ms": 73.756, "count": 23, "max_ms": 88.048, "p95_ms": 83.203}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.075, "count": 23, "max_ms": 96.331, "p95_ms": 93.518}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.502, "count": 23, "max_ms": 111.131, "p95_ms": 108.911}, "read_interval": {"avg_ms": 1302.261, "count": 23, "max_ms": 16118.951, "p95_ms": 11584.833}, "triton_wait": {"avg_ms": 82.5, "count": 23, "max_ms": 98.237, "p95_ms": 96.343}, "visualize": {"avg_ms": 3.608, "count": 23, "max_ms": 3.876, "p95_ms": 3.779}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2344}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6003.074, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:54:31+09:00", "last_error_summary": "[h264 @ 0x558f8491a180] error while decoding MB 0 28, bytestream -9 [in#0/rtsp @ 0x558f84892e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558f84892e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:54:31+09:00", "last_error_summary": "[h264 @ 0x558f8491a180] error while decoding MB 0 28, bytestream -9 [in#0/rtsp @ 0x558f84892e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558f84892e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:54:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.451, "last_started_at": "2026-07-06T21:51:15+09:00", "pid": 30638, "process": {"exists": false, "pid": 30638}, "restart_count": 67, "restart_count_per_hour": 17, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.23, "sample_count": 2132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281388, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:54:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T21:54:31+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T21:54:31+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-06T21:54:31.657951+09:00", "mtime_epoch": 1783342471.6579514, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106430}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:54:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.094, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 24, "pushed_frames": 24, "read_frames": 23, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.801, "pushed": 0.801, "read": 0.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.325, "count": 24, "max_ms": 5.592, "p95_ms": 5.568}, "gstreamer_push": {"avg_ms": 1.311, "count": 24, "max_ms": 2.155, "p95_ms": 1.482}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.308, "count": 24, "max_ms": 81.252, "p95_ms": 81.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.929, "count": 24, "max_ms": 98.225, "p95_ms": 94.535}, "yolo11_640_fire_11class": {"avg_ms": 74.16, "count": 24, "max_ms": 88.048, "p95_ms": 83.434}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.46, "count": 24, "max_ms": 96.331, "p95_ms": 93.518}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.849, "count": 24, "max_ms": 111.131, "p95_ms": 108.911}, "read_interval": {"avg_ms": 1302.261, "count": 23, "max_ms": 16118.951, "p95_ms": 11584.833}, "triton_wait": {"avg_ms": 82.868, "count": 24, "max_ms": 98.237, "p95_ms": 96.343}, "visualize": {"avg_ms": 3.607, "count": 24, "max_ms": 3.876, "p95_ms": 3.779}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2344}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6003.113, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:54:31+09:00", "last_error_summary": "[h264 @ 0x558f8491a180] error while decoding MB 0 28, bytestream -9 [in#0/rtsp @ 0x558f84892e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558f84892e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:54:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.458, "last_started_at": "2026-07-06T21:51:15+09:00", "pid": 30638, "process": {"exists": false, "pid": 30638}, "restart_count": 67, "restart_count_per_hour": 17, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.23, "sample_count": 2132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281388, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:54:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T21:54:31+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T21:54:31+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T21:54:31.657951+09:00", "mtime_epoch": 1783342471.6579514, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106430}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:54:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.094, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 24, "pushed_frames": 24, "read_frames": 23, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.801, "pushed": 0.801, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.325, "count": 24, "max_ms": 5.592, "p95_ms": 5.568}, "gstreamer_push": {"avg_ms": 1.311, "count": 24, "max_ms": 2.155, "p95_ms": 1.482}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.308, "count": 24, "max_ms": 81.252, "p95_ms": 81.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.929, "count": 24, "max_ms": 98.225, "p95_ms": 94.535}, "yolo11_640_fire_11class": {"avg_ms": 74.16, "count": 24, "max_ms": 88.048, "p95_ms": 83.434}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.46, "count": 24, "max_ms": 96.331, "p95_ms": 93.518}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.849, "count": 24, "max_ms": 111.131, "p95_ms": 108.911}, "read_interval": {"avg_ms": 1302.261, "count": 23, "max_ms": 16118.951, "p95_ms": 11584.833}, "triton_wait": {"avg_ms": 82.868, "count": 24, "max_ms": 98.237, "p95_ms": 96.343}, "visualize": {"avg_ms": 3.607, "count": 24, "max_ms": 3.876, "p95_ms": 3.779}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2344}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6023.785, "limit_mb": 8796093022207.996}, "detail": {"pid": 30897, "restart_count": 68, "start_count": 69}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:54:31+09:00", "last_error_summary": "[h264 @ 0x558f8491a180] error while decoding MB 0 28, bytestream -9 [in#0/rtsp @ 0x558f84892e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558f84892e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:54:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:54:34+09:00", "pid": 30897, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.684, "name": "ffmpeg", "pid": 30897, "rss_mb": 57.684, "state": "S (sleeping)", "threads": 1}, "restart_count": 68, "restart_count_per_hour": 18, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.82, "sample_count": 2089, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281388, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:54:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.898, "last_frame_pushed_at": "2026-07-06T21:54:31+09:00", "last_frame_read_age_sec": 5.004, "last_frame_read_at": "2026-07-06T21:54:31+09:00", "mjpeg": {"age_sec": 4.885, "exists": true, "mtime": "2026-07-06T21:54:31.769953+09:00", "mtime_epoch": 1783342471.7699535, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101366}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:54:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.094, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.625, "pushed": 0.625, "read": 0.625}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.33, "count": 14, "max_ms": 5.592, "p95_ms": 5.592}, "gstreamer_push": {"avg_ms": 1.259, "count": 14, "max_ms": 1.416, "p95_ms": 1.416}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.004, "count": 14, "max_ms": 81.189, "p95_ms": 81.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.39, "count": 14, "max_ms": 98.225, "p95_ms": 98.225}, "yolo11_640_fire_11class": {"avg_ms": 75.217, "count": 14, "max_ms": 88.048, "p95_ms": 88.048}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.079, "count": 14, "max_ms": 96.331, "p95_ms": 96.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.607, "count": 14, "max_ms": 111.131, "p95_ms": 111.131}, "read_interval": {"avg_ms": 2069.787, "count": 14, "max_ms": 16118.951, "p95_ms": 16118.951}, "triton_wait": {"avg_ms": 82.73, "count": 14, "max_ms": 98.237, "p95_ms": 98.237}, "visualize": {"avg_ms": 3.585, "count": 14, "max_ms": 3.73, "p95_ms": 3.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.397, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2352}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6311.18, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:54:31+09:00", "last_error_summary": "[h264 @ 0x558f8491a180] error while decoding MB 0 28, bytestream -9 [in#0/rtsp @ 0x558f84892e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558f84892e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:54:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.815, "last_started_at": "2026-07-06T21:54:34+09:00", "pid": 30897, "process": {"exists": true, "fd_count": 5, "hwm_mb": 326.793, "name": "ffmpeg", "pid": 30897, "rss_mb": 326.793, "state": "S (sleeping)", "threads": 106}, "restart_count": 68, "restart_count_per_hour": 17, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.678, "sample_count": 2132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 282268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:56:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.058, "last_frame_pushed_at": "2026-07-06T21:56:14+09:00", "last_frame_read_age_sec": 30.147, "last_frame_read_at": "2026-07-06T21:56:14+09:00", "mjpeg": {"age_sec": 30.041, "exists": true, "mtime": "2026-07-06T21:56:14.427831+09:00", "mtime_epoch": 1783342574.4278312, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96956}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:56:44+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2352}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6303.203, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:54:31+09:00", "last_error_summary": "[h264 @ 0x558f8491a180] error while decoding MB 0 28, bytestream -9 [in#0/rtsp @ 0x558f84892e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558f84892e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:54:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.827, "last_started_at": "2026-07-06T21:54:34+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 68, "restart_count_per_hour": 17, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.678, "sample_count": 2132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 282268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:56:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.07, "last_frame_pushed_at": "2026-07-06T21:56:14+09:00", "last_frame_read_age_sec": 30.159, "last_frame_read_at": "2026-07-06T21:56:14+09:00", "mjpeg": {"age_sec": 30.053, "exists": true, "mtime": "2026-07-06T21:56:14.427831+09:00", "mtime_epoch": 1783342574.4278312, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96956}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:56:44+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2352}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6012.898, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ea87e6d240] error while decoding MB 67 25, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:56:44+09:00", "last_error_summary": "[h264 @ 0x55ea87e6d240] error while decoding MB 67 25, bytestream -5", "last_exit_at": "2026-07-06T21:54:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.878, "last_started_at": "2026-07-06T21:54:34+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 68, "restart_count_per_hour": 17, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.057, "sample_count": 2131, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 282268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:56:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.121, "last_frame_pushed_at": "2026-07-06T21:56:14+09:00", "last_frame_read_age_sec": 30.21, "last_frame_read_at": "2026-07-06T21:56:14+09:00", "mjpeg": {"age_sec": 30.104, "exists": true, "mtime": "2026-07-06T21:56:14.427831+09:00", "mtime_epoch": 1783342574.4278312, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96956}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:56:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2352}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6079.535, "limit_mb": 8796093022207.996}, "detail": {"pid": 31049, "restart_count": 69, "start_count": 70}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:56:44+09:00", "last_error_summary": "[h264 @ 0x55ea87e6d240] error while decoding MB 67 25, bytestream -5", "last_exit_at": "2026-07-06T21:54:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:56:46+09:00", "pid": 31049, "process": {"exists": true, "fd_count": 5, "hwm_mb": 101.461, "name": "ffmpeg", "pid": 31049, "rss_mb": 101.461, "state": "S (sleeping)", "threads": 19}, "restart_count": 69, "restart_count_per_hour": 18, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.613, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 282268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:56:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.544, "last_frame_pushed_at": "2026-07-06T21:56:14+09:00", "last_frame_read_age_sec": 34.633, "last_frame_read_at": "2026-07-06T21:56:14+09:00", "mjpeg": {"age_sec": 34.527, "exists": true, "mtime": "2026-07-06T21:56:14.427831+09:00", "mtime_epoch": 1783342574.4278312, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96956}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:56:48+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2359}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6022.754, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x561f23a60200] missing picture in access unit with size 6 [h264 @ 0x561f24260dc0] no frame! [in#0/rtsp @ 0x561f23a5be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f23a5be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561f23a60840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:58:55+09:00", "last_error_summary": "[NULL @ 0x561f23a60200] missing picture in access unit with size 6 [h264 @ 0x561f24260dc0] no frame! [in#0/rtsp @ 0x561f23a5be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f23a5be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561f23a60840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:54:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.922, "last_started_at": "2026-07-06T21:56:46+09:00", "pid": 31049, "process": {"exists": false, "pid": 31049}, "restart_count": 69, "restart_count_per_hour": 18, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.659, "sample_count": 1791, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 283103, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:58:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-06T21:58:55+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T21:58:55+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-06T21:58:55.818788+09:00", "mtime_epoch": 1783342735.8187878, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102781}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:58:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.434, "pushed": 0.434, "read": 0.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.039, "count": 12, "max_ms": 4.212, "p95_ms": 4.212}, "gstreamer_push": {"avg_ms": 1.244, "count": 12, "max_ms": 1.307, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.944, "count": 12, "max_ms": 80.985, "p95_ms": 80.985}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.908, "count": 12, "max_ms": 85.467, "p95_ms": 85.467}, "yolo11_640_fire_11class": {"avg_ms": 69.666, "count": 12, "max_ms": 79.073, "p95_ms": 79.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.567, "count": 12, "max_ms": 87.647, "p95_ms": 87.647}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.037, "count": 12, "max_ms": 100.14, "p95_ms": 100.14}, "read_interval": {"avg_ms": 2625.428, "count": 13, "max_ms": 26445.912, "p95_ms": 26445.912}, "triton_wait": {"avg_ms": 77.6, "count": 12, "max_ms": 87.658, "p95_ms": 87.658}, "visualize": {"avg_ms": 3.524, "count": 12, "max_ms": 3.577, "p95_ms": 3.577}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.634, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2359}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6022.453, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:58:55+09:00", "last_error_summary": "[NULL @ 0x561f23a60200] missing picture in access unit with size 6 [h264 @ 0x561f24260dc0] no frame! [in#0/rtsp @ 0x561f23a5be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f23a5be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561f23a60840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:58:55+09:00", "last_error_summary": "[NULL @ 0x561f23a60200] missing picture in access unit with size 6 [h264 @ 0x561f24260dc0] no frame! [in#0/rtsp @ 0x561f23a5be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f23a5be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561f23a60840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:58:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.939, "last_started_at": "2026-07-06T21:56:46+09:00", "pid": 31049, "process": {"exists": false, "pid": 31049}, "restart_count": 69, "restart_count_per_hour": 18, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.659, "sample_count": 1792, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 283104, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:58:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T21:58:55+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-06T21:58:55+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-06T21:58:55.818788+09:00", "mtime_epoch": 1783342735.8187878, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102781}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:58:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.867, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.47, "pushed": 0.47, "read": 0.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.038, "count": 13, "max_ms": 4.212, "p95_ms": 4.212}, "gstreamer_push": {"avg_ms": 1.244, "count": 13, "max_ms": 1.307, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.651, "count": 13, "max_ms": 80.985, "p95_ms": 80.985}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.441, "count": 13, "max_ms": 85.467, "p95_ms": 85.467}, "yolo11_640_fire_11class": {"avg_ms": 68.612, "count": 13, "max_ms": 79.073, "p95_ms": 79.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.06, "count": 13, "max_ms": 87.647, "p95_ms": 87.647}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.631, "count": 13, "max_ms": 100.14, "p95_ms": 100.14}, "read_interval": {"avg_ms": 2625.428, "count": 13, "max_ms": 26445.912, "p95_ms": 26445.912}, "triton_wait": {"avg_ms": 75.938, "count": 13, "max_ms": 87.658, "p95_ms": 87.658}, "visualize": {"avg_ms": 3.691, "count": 13, "max_ms": 5.693, "p95_ms": 5.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.651, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2359}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6022.438, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:58:55+09:00", "last_error_summary": "[NULL @ 0x561f23a60200] missing picture in access unit with size 6 [h264 @ 0x561f24260dc0] no frame! [in#0/rtsp @ 0x561f23a5be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f23a5be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561f23a60840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:58:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.946, "last_started_at": "2026-07-06T21:56:46+09:00", "pid": 31049, "process": {"exists": false, "pid": 31049}, "restart_count": 69, "restart_count_per_hour": 18, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.659, "sample_count": 1792, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 283104, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:58:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T21:58:55+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-06T21:58:55+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-06T21:58:55.818788+09:00", "mtime_epoch": 1783342735.8187878, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102781}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:58:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.867, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.47, "pushed": 0.47, "read": 0.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.038, "count": 13, "max_ms": 4.212, "p95_ms": 4.212}, "gstreamer_push": {"avg_ms": 1.244, "count": 13, "max_ms": 1.307, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.651, "count": 13, "max_ms": 80.985, "p95_ms": 80.985}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.441, "count": 13, "max_ms": 85.467, "p95_ms": 85.467}, "yolo11_640_fire_11class": {"avg_ms": 68.612, "count": 13, "max_ms": 79.073, "p95_ms": 79.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.06, "count": 13, "max_ms": 87.647, "p95_ms": 87.647}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.631, "count": 13, "max_ms": 100.14, "p95_ms": 100.14}, "read_interval": {"avg_ms": 2625.428, "count": 13, "max_ms": 26445.912, "p95_ms": 26445.912}, "triton_wait": {"avg_ms": 75.938, "count": 13, "max_ms": 87.658, "p95_ms": 87.658}, "visualize": {"avg_ms": 3.691, "count": 13, "max_ms": 5.693, "p95_ms": 5.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.657, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2359}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6202.922, "limit_mb": 8796093022207.996}, "detail": {"pid": 31201, "restart_count": 70, "start_count": 71}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:58:55+09:00", "last_error_summary": "[NULL @ 0x561f23a60200] missing picture in access unit with size 6 [h264 @ 0x561f24260dc0] no frame! [in#0/rtsp @ 0x561f23a5be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561f23a5be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561f23a60840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T21:58:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:58:58+09:00", "pid": 31201, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.504, "name": "ffmpeg", "pid": 31201, "rss_mb": 206.504, "state": "S (sleeping)", "threads": 106}, "restart_count": 70, "restart_count_per_hour": 18, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.332, "sample_count": 1753, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 283104, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:58:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.447, "last_frame_pushed_at": "2026-07-06T21:58:55+09:00", "last_frame_read_age_sec": 4.52, "last_frame_read_at": "2026-07-06T21:58:55+09:00", "mjpeg": {"age_sec": 4.432, "exists": true, "mtime": "2026-07-06T21:58:55.902789+09:00", "mtime_epoch": 1783342735.9027894, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103861}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T21:59:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1547.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.125, "pushed": 2.125, "read": 2.125}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.036, "count": 12, "max_ms": 4.212, "p95_ms": 4.212}, "gstreamer_push": {"avg_ms": 1.239, "count": 12, "max_ms": 1.296, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.834, "count": 12, "max_ms": 80.985, "p95_ms": 80.985}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.58, "count": 12, "max_ms": 85.467, "p95_ms": 85.467}, "yolo11_640_fire_11class": {"avg_ms": 69.616, "count": 12, "max_ms": 79.073, "p95_ms": 79.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.476, "count": 12, "max_ms": 87.647, "p95_ms": 87.647}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.076, "count": 12, "max_ms": 100.14, "p95_ms": 100.14}, "read_interval": {"avg_ms": 2297.664, "count": 12, "max_ms": 26445.912, "p95_ms": 26445.912}, "triton_wait": {"avg_ms": 77.367, "count": 12, "max_ms": 87.658, "p95_ms": 87.658}, "visualize": {"avg_ms": 3.703, "count": 12, "max_ms": 5.693, "p95_ms": 5.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.647, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2164}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6072.715, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563f3e9feb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563f3e9feb00] error while decoding MB 0 39, bytestream 4496 [h264 @ 0x563f3de52840] cabac decode of qscale diff failed at 17 17 [h264 @ 0x563f3de52840] error while decoding MB 17 17, bytestream 2059 [h264 @ 0x563f3ddfdcc0] error while decoding MB 15 39, bytestream -15 [in#0/rtsp @ 0x563f3ddcee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f3ddcee00] Error retrieving a packet from demuxer: C", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 118, "h264_decode_error_count": 385, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:59:42+09:00", "last_error_summary": "[h264 @ 0x563f3e9feb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563f3e9feb00] error while decoding MB 0 39, bytestream 4496 [h264 @ 0x563f3de52840] cabac decode of qscale diff failed at 17 17 [h264 @ 0x563f3de52840] error while decoding MB 17 17, bytestream 2059 [h264 @ 0x563f3ddfdcc0] error while decoding MB 15 39, bytestream -15 [in#0/rtsp @ 0x563f3ddcee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f3ddcee00] Error retrieving a packet from demuxer: C", "last_exit_at": "2026-07-06T21:51:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 490.374, "last_started_at": "2026-07-06T21:51:32+09:00", "pid": 30790, "process": {"exists": false, "pid": 30790}, "restart_count": 127, "restart_count_per_hour": 7, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 128}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.602, "sample_count": 2696, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 259766, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:59:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T21:59:42+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T21:59:42+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-06T21:59:42.519644+09:00", "mtime_epoch": 1783342782.5196443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 111414}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:59:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1661.578, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.337, "pushed": 3.337, "read": 3.337}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.547, "count": 100, "max_ms": 5.582, "p95_ms": 4.118}, "gstreamer_push": {"avg_ms": 1.322, "count": 100, "max_ms": 1.547, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.256, "count": 100, "max_ms": 85.658, "p95_ms": 80.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.304, "count": 100, "max_ms": 113.283, "p95_ms": 106.407}, "yolo11_640_fire_11class": {"avg_ms": 73.901, "count": 100, "max_ms": 90.939, "p95_ms": 86.673}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.118, "count": 100, "max_ms": 100.527, "p95_ms": 95.258}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.918, "count": 100, "max_ms": 125.84, "p95_ms": 118.869}, "read_interval": {"avg_ms": 299.974, "count": 100, "max_ms": 19168.663, "p95_ms": 134.057}, "triton_wait": {"avg_ms": 83.624, "count": 100, "max_ms": 113.297, "p95_ms": 106.42}, "visualize": {"avg_ms": 3.647, "count": 100, "max_ms": 4.384, "p95_ms": 3.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2164}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6072.223, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T21:59:42+09:00", "last_error_summary": "[h264 @ 0x563f3e9feb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563f3e9feb00] error while decoding MB 0 39, bytestream 4496 [h264 @ 0x563f3de52840] cabac decode of qscale diff failed at 17 17 [h264 @ 0x563f3de52840] error while decoding MB 17 17, bytestream 2059 [h264 @ 0x563f3ddfdcc0] error while decoding MB 15 39, bytestream -15 [in#0/rtsp @ 0x563f3ddcee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f3ddcee00] Error retrieving a packet from demuxer: C", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 118, "h264_decode_error_count": 385, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:59:42+09:00", "last_error_summary": "[h264 @ 0x563f3e9feb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563f3e9feb00] error while decoding MB 0 39, bytestream 4496 [h264 @ 0x563f3de52840] cabac decode of qscale diff failed at 17 17 [h264 @ 0x563f3de52840] error while decoding MB 17 17, bytestream 2059 [h264 @ 0x563f3ddfdcc0] error while decoding MB 15 39, bytestream -15 [in#0/rtsp @ 0x563f3ddcee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f3ddcee00] Error retrieving a packet from demuxer: C", "last_exit_at": "2026-07-06T21:59:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 490.436, "last_started_at": "2026-07-06T21:51:32+09:00", "pid": 30790, "process": {"exists": false, "pid": 30790}, "restart_count": 127, "restart_count_per_hour": 7, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 128}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.602, "sample_count": 2697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 259767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:59:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T21:59:42+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T21:59:42+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-06T21:59:42.519644+09:00", "mtime_epoch": 1783342782.5196443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 111414}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:59:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1661.578, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.342, "pushed": 3.342, "read": 3.309}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.548, "count": 100, "max_ms": 5.582, "p95_ms": 4.118}, "gstreamer_push": {"avg_ms": 1.321, "count": 100, "max_ms": 1.547, "p95_ms": 1.421}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.479, "count": 100, "max_ms": 85.658, "p95_ms": 80.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.495, "count": 100, "max_ms": 113.283, "p95_ms": 106.407}, "yolo11_640_fire_11class": {"avg_ms": 74.089, "count": 100, "max_ms": 90.939, "p95_ms": 86.673}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.304, "count": 100, "max_ms": 100.527, "p95_ms": 95.258}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.105, "count": 100, "max_ms": 125.84, "p95_ms": 118.869}, "read_interval": {"avg_ms": 302.026, "count": 99, "max_ms": 19168.663, "p95_ms": 135.238}, "triton_wait": {"avg_ms": 83.81, "count": 100, "max_ms": 113.297, "p95_ms": 106.42}, "visualize": {"avg_ms": 3.648, "count": 100, "max_ms": 4.384, "p95_ms": 3.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2164}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6072.25, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 118, "h264_decode_error_count": 385, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:59:42+09:00", "last_error_summary": "[h264 @ 0x563f3e9feb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563f3e9feb00] error while decoding MB 0 39, bytestream 4496 [h264 @ 0x563f3de52840] cabac decode of qscale diff failed at 17 17 [h264 @ 0x563f3de52840] error while decoding MB 17 17, bytestream 2059 [h264 @ 0x563f3ddfdcc0] error while decoding MB 15 39, bytestream -15 [in#0/rtsp @ 0x563f3ddcee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f3ddcee00] Error retrieving a packet from demuxer: C", "last_exit_at": "2026-07-06T21:59:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 490.447, "last_started_at": "2026-07-06T21:51:32+09:00", "pid": 30790, "process": {"exists": false, "pid": 30790}, "restart_count": 127, "restart_count_per_hour": 7, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 128}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.602, "sample_count": 2697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 259767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:59:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.022, "last_frame_pushed_at": "2026-07-06T21:59:42+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-06T21:59:42+09:00", "mjpeg": {"age_sec": 0.003, "exists": true, "mtime": "2026-07-06T21:59:42.631646+09:00", "mtime_epoch": 1783342782.6316464, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:59:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1661.578, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.341, "pushed": 3.341, "read": 3.308}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.548, "count": 100, "max_ms": 5.582, "p95_ms": 4.118}, "gstreamer_push": {"avg_ms": 1.321, "count": 100, "max_ms": 1.547, "p95_ms": 1.421}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.479, "count": 100, "max_ms": 85.658, "p95_ms": 80.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.495, "count": 100, "max_ms": 113.283, "p95_ms": 106.407}, "yolo11_640_fire_11class": {"avg_ms": 74.089, "count": 100, "max_ms": 90.939, "p95_ms": 86.673}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.304, "count": 100, "max_ms": 100.527, "p95_ms": 95.258}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.105, "count": 100, "max_ms": 125.84, "p95_ms": 118.869}, "read_interval": {"avg_ms": 302.026, "count": 99, "max_ms": 19168.663, "p95_ms": 135.238}, "triton_wait": {"avg_ms": 83.81, "count": 100, "max_ms": 113.297, "p95_ms": 106.42}, "visualize": {"avg_ms": 3.648, "count": 100, "max_ms": 4.384, "p95_ms": 3.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2164}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6181.148, "limit_mb": 8796093022207.996}, "detail": {"pid": 31353, "restart_count": 128, "start_count": 129}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 118, "h264_decode_error_count": 385, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T21:59:42+09:00", "last_error_summary": "[h264 @ 0x563f3e9feb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563f3e9feb00] error while decoding MB 0 39, bytestream 4496 [h264 @ 0x563f3de52840] cabac decode of qscale diff failed at 17 17 [h264 @ 0x563f3de52840] error while decoding MB 17 17, bytestream 2059 [h264 @ 0x563f3ddfdcc0] error while decoding MB 15 39, bytestream -15 [in#0/rtsp @ 0x563f3ddcee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f3ddcee00] Error retrieving a packet from demuxer: C", "last_exit_at": "2026-07-06T21:59:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T21:59:45+09:00", "pid": 31353, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.93, "name": "ffmpeg", "pid": 31353, "rss_mb": 138.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 128, "restart_count_per_hour": 8, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 129}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.486, "sample_count": 2651, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 259767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T21:59:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.999, "last_frame_pushed_at": "2026-07-06T21:59:42+09:00", "last_frame_read_age_sec": 5.104, "last_frame_read_at": "2026-07-06T21:59:42+09:00", "mjpeg": {"age_sec": 4.981, "exists": true, "mtime": "2026-07-06T21:59:42.631646+09:00", "mtime_epoch": 1783342782.6316464, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T21:59:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1661.578, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 55, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.834, "pushed": 1.834, "read": 1.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.601, "count": 55, "max_ms": 5.582, "p95_ms": 4.342}, "gstreamer_push": {"avg_ms": 1.33, "count": 55, "max_ms": 1.547, "p95_ms": 1.438}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.566, "count": 55, "max_ms": 82.681, "p95_ms": 79.916}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.627, "count": 55, "max_ms": 106.407, "p95_ms": 92.372}, "yolo11_640_fire_11class": {"avg_ms": 73.242, "count": 55, "max_ms": 87.0, "p95_ms": 85.375}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.09, "count": 55, "max_ms": 97.087, "p95_ms": 93.894}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.632, "count": 55, "max_ms": 118.869, "p95_ms": 109.99}, "read_interval": {"avg_ms": 454.684, "count": 55, "max_ms": 19168.663, "p95_ms": 127.329}, "triton_wait": {"avg_ms": 82.276, "count": 55, "max_ms": 106.42, "p95_ms": 97.101}, "visualize": {"avg_ms": 3.668, "count": 55, "max_ms": 4.384, "p95_ms": 4.144}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2373}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6014.051, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562b2872c000] error while decoding MB 78 31, bytestream -7 [in#0/rtsp @ 0x562b27bdee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b27bdee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:02:35+09:00", "last_error_summary": "[h264 @ 0x562b2872c000] error while decoding MB 78 31, bytestream -7 [in#0/rtsp @ 0x562b27bdee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b27bdee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T21:58:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 216.91, "last_started_at": "2026-07-06T21:58:58+09:00", "pid": 31201, "process": {"exists": false, "pid": 31201}, "restart_count": 70, "restart_count_per_hour": 17, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.267, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 284796, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:02:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T22:02:35+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-06T22:02:35+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-06T22:02:35.190814+09:00", "mtime_epoch": 1783342955.1908138, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101564}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:02:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1548.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.788}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.304, "count": 13, "max_ms": 5.594, "p95_ms": 5.594}, "gstreamer_push": {"avg_ms": 1.283, "count": 13, "max_ms": 1.401, "p95_ms": 1.401}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.732, "count": 13, "max_ms": 79.27, "p95_ms": 79.27}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.641, "count": 13, "max_ms": 85.314, "p95_ms": 85.314}, "yolo11_640_fire_11class": {"avg_ms": 72.932, "count": 13, "max_ms": 80.536, "p95_ms": 80.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.256, "count": 13, "max_ms": 92.073, "p95_ms": 92.073}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.972, "count": 13, "max_ms": 104.835, "p95_ms": 104.835}, "read_interval": {"avg_ms": 2406.037, "count": 14, "max_ms": 16424.794, "p95_ms": 16424.794}, "triton_wait": {"avg_ms": 81.99, "count": 13, "max_ms": 92.085, "p95_ms": 92.085}, "visualize": {"avg_ms": 3.655, "count": 13, "max_ms": 4.208, "p95_ms": 4.208}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.777, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2373}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6013.309, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:02:35+09:00", "last_error_summary": "[h264 @ 0x562b2872c000] error while decoding MB 78 31, bytestream -7 [in#0/rtsp @ 0x562b27bdee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b27bdee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:02:35+09:00", "last_error_summary": "[h264 @ 0x562b2872c000] error while decoding MB 78 31, bytestream -7 [in#0/rtsp @ 0x562b27bdee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b27bdee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:02:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 216.954, "last_started_at": "2026-07-06T21:58:58+09:00", "pid": 31201, "process": {"exists": false, "pid": 31201}, "restart_count": 70, "restart_count_per_hour": 17, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.267, "sample_count": 2119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 284797, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:02:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:02:35+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-06T22:02:35+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-06T22:02:35.190814+09:00", "mtime_epoch": 1783342955.1908138, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101564}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:02:35+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1548.641, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.786, "pushed": 0.786, "read": 0.786}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.289, "count": 14, "max_ms": 5.594, "p95_ms": 5.594}, "gstreamer_push": {"avg_ms": 1.292, "count": 14, "max_ms": 1.409, "p95_ms": 1.409}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.497, "count": 14, "max_ms": 79.27, "p95_ms": 79.27}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.698, "count": 14, "max_ms": 85.314, "p95_ms": 85.314}, "yolo11_640_fire_11class": {"avg_ms": 73.181, "count": 14, "max_ms": 80.536, "p95_ms": 80.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.526, "count": 14, "max_ms": 92.073, "p95_ms": 92.073}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.119, "count": 14, "max_ms": 104.835, "p95_ms": 104.835}, "read_interval": {"avg_ms": 2406.037, "count": 14, "max_ms": 16424.794, "p95_ms": 16424.794}, "triton_wait": {"avg_ms": 82.138, "count": 14, "max_ms": 92.085, "p95_ms": 92.085}, "visualize": {"avg_ms": 3.655, "count": 14, "max_ms": 4.208, "p95_ms": 4.208}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.821, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2373}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6013.289, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:02:35+09:00", "last_error_summary": "[h264 @ 0x562b2872c000] error while decoding MB 78 31, bytestream -7 [in#0/rtsp @ 0x562b27bdee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b27bdee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:02:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 216.961, "last_started_at": "2026-07-06T21:58:58+09:00", "pid": 31201, "process": {"exists": false, "pid": 31201}, "restart_count": 70, "restart_count_per_hour": 17, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.267, "sample_count": 2119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 284797, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:02:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T22:02:35+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T22:02:35+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-06T22:02:35.190814+09:00", "mtime_epoch": 1783342955.1908138, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101564}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:02:35+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1548.641, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.785, "pushed": 0.785, "read": 0.785}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.289, "count": 14, "max_ms": 5.594, "p95_ms": 5.594}, "gstreamer_push": {"avg_ms": 1.292, "count": 14, "max_ms": 1.409, "p95_ms": 1.409}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.497, "count": 14, "max_ms": 79.27, "p95_ms": 79.27}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.698, "count": 14, "max_ms": 85.314, "p95_ms": 85.314}, "yolo11_640_fire_11class": {"avg_ms": 73.181, "count": 14, "max_ms": 80.536, "p95_ms": 80.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.526, "count": 14, "max_ms": 92.073, "p95_ms": 92.073}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.119, "count": 14, "max_ms": 104.835, "p95_ms": 104.835}, "read_interval": {"avg_ms": 2406.037, "count": 14, "max_ms": 16424.794, "p95_ms": 16424.794}, "triton_wait": {"avg_ms": 82.138, "count": 14, "max_ms": 92.085, "p95_ms": 92.085}, "visualize": {"avg_ms": 3.655, "count": 14, "max_ms": 4.208, "p95_ms": 4.208}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.828, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2373}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6145.492, "limit_mb": 8796093022207.996}, "detail": {"pid": 31460, "restart_count": 71, "start_count": 72}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:02:35+09:00", "last_error_summary": "[h264 @ 0x562b2872c000] error while decoding MB 78 31, bytestream -7 [in#0/rtsp @ 0x562b27bdee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b27bdee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:02:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:02:38+09:00", "pid": 31460, "process": {"exists": true, "fd_count": 5, "hwm_mb": 159.746, "name": "ffmpeg", "pid": 31460, "rss_mb": 159.746, "state": "R (running)", "threads": 71}, "restart_count": 71, "restart_count_per_hour": 18, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.301, "sample_count": 2076, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 284797, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:02:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.922, "last_frame_pushed_at": "2026-07-06T22:02:35+09:00", "last_frame_read_age_sec": 5.021, "last_frame_read_at": "2026-07-06T22:02:35+09:00", "mjpeg": {"age_sec": 4.906, "exists": true, "mtime": "2026-07-06T22:02:35.298816+09:00", "mtime_epoch": 1783342955.2988157, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101223}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:02:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1548.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.616, "pushed": 0.616, "read": 0.616}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.289, "count": 14, "max_ms": 5.594, "p95_ms": 5.594}, "gstreamer_push": {"avg_ms": 1.292, "count": 14, "max_ms": 1.409, "p95_ms": 1.409}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.497, "count": 14, "max_ms": 79.27, "p95_ms": 79.27}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.698, "count": 14, "max_ms": 85.314, "p95_ms": 85.314}, "yolo11_640_fire_11class": {"avg_ms": 73.181, "count": 14, "max_ms": 80.536, "p95_ms": 80.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.526, "count": 14, "max_ms": 92.073, "p95_ms": 92.073}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.119, "count": 14, "max_ms": 104.835, "p95_ms": 104.835}, "read_interval": {"avg_ms": 2406.037, "count": 14, "max_ms": 16424.794, "p95_ms": 16424.794}, "triton_wait": {"avg_ms": 82.138, "count": 14, "max_ms": 92.085, "p95_ms": 92.085}, "visualize": {"avg_ms": 3.655, "count": 14, "max_ms": 4.208, "p95_ms": 4.208}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.737, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2382}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6024.621, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563a468e6cc0] error while decoding MB 79 30, bytestream -5 [in#0/rtsp @ 0x563a45da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a45da3e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:05:16+09:00", "last_error_summary": "[h264 @ 0x563a468e6cc0] error while decoding MB 79 30, bytestream -5 [in#0/rtsp @ 0x563a45da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a45da3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:02:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.194, "last_started_at": "2026-07-06T22:02:38+09:00", "pid": 31460, "process": {"exists": false, "pid": 31460}, "restart_count": 71, "restart_count_per_hour": 17, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.965, "sample_count": 2079, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 285926, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:05:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T22:05:16+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T22:05:16+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T22:05:16.337775+09:00", "mtime_epoch": 1783343116.3377748, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 112643}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:05:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1549.414, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.739, "pushed": 0.739, "read": 0.796}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.163, "count": 13, "max_ms": 5.862, "p95_ms": 5.862}, "gstreamer_push": {"avg_ms": 1.267, "count": 13, "max_ms": 1.341, "p95_ms": 1.341}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.086, "count": 13, "max_ms": 83.659, "p95_ms": 83.659}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.113, "count": 13, "max_ms": 88.025, "p95_ms": 88.025}, "yolo11_640_fire_11class": {"avg_ms": 66.013, "count": 13, "max_ms": 86.079, "p95_ms": 86.079}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.134, "count": 13, "max_ms": 97.966, "p95_ms": 97.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.801, "count": 13, "max_ms": 110.793, "p95_ms": 110.793}, "read_interval": {"avg_ms": 2240.977, "count": 14, "max_ms": 16376.982, "p95_ms": 16376.982}, "triton_wait": {"avg_ms": 73.062, "count": 13, "max_ms": 97.978, "p95_ms": 97.978}, "visualize": {"avg_ms": 3.605, "count": 13, "max_ms": 4.099, "p95_ms": 4.099}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.593, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2382}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6023.641, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:05:16+09:00", "last_error_summary": "[h264 @ 0x563a468e6cc0] error while decoding MB 79 30, bytestream -5 [in#0/rtsp @ 0x563a45da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a45da3e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:05:16+09:00", "last_error_summary": "[h264 @ 0x563a468e6cc0] error while decoding MB 79 30, bytestream -5 [in#0/rtsp @ 0x563a45da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a45da3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:05:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.273, "last_started_at": "2026-07-06T22:02:38+09:00", "pid": 31460, "process": {"exists": false, "pid": 31460}, "restart_count": 71, "restart_count_per_hour": 17, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.007, "sample_count": 2079, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 285927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:05:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:05:16+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-06T22:05:16+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-06T22:05:16.337775+09:00", "mtime_epoch": 1783343116.3377748, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 112643}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:05:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1549.414, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.792, "pushed": 0.792, "read": 0.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.161, "count": 14, "max_ms": 5.862, "p95_ms": 5.862}, "gstreamer_push": {"avg_ms": 1.273, "count": 14, "max_ms": 1.345, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.996, "count": 14, "max_ms": 83.822, "p95_ms": 83.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.623, "count": 14, "max_ms": 119.252, "p95_ms": 119.252}, "yolo11_640_fire_11class": {"avg_ms": 67.322, "count": 14, "max_ms": 86.079, "p95_ms": 86.079}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.707, "count": 14, "max_ms": 97.966, "p95_ms": 97.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.126, "count": 14, "max_ms": 132.352, "p95_ms": 132.352}, "read_interval": {"avg_ms": 2240.977, "count": 14, "max_ms": 16376.982, "p95_ms": 16376.982}, "triton_wait": {"avg_ms": 76.363, "count": 14, "max_ms": 119.267, "p95_ms": 119.267}, "visualize": {"avg_ms": 3.61, "count": 14, "max_ms": 4.099, "p95_ms": 4.099}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.673, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2382}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6023.547, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:05:16+09:00", "last_error_summary": "[h264 @ 0x563a468e6cc0] error while decoding MB 79 30, bytestream -5 [in#0/rtsp @ 0x563a45da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a45da3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:05:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.284, "last_started_at": "2026-07-06T22:02:38+09:00", "pid": 31460, "process": {"exists": false, "pid": 31460}, "restart_count": 71, "restart_count_per_hour": 17, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.007, "sample_count": 2079, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 285927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:05:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-06T22:05:16+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-06T22:05:16+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-06T22:05:16.337775+09:00", "mtime_epoch": 1783343116.3377748, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 112643}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:05:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1549.414, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.792, "pushed": 0.792, "read": 0.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.161, "count": 14, "max_ms": 5.862, "p95_ms": 5.862}, "gstreamer_push": {"avg_ms": 1.273, "count": 14, "max_ms": 1.345, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.996, "count": 14, "max_ms": 83.822, "p95_ms": 83.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.623, "count": 14, "max_ms": 119.252, "p95_ms": 119.252}, "yolo11_640_fire_11class": {"avg_ms": 67.322, "count": 14, "max_ms": 86.079, "p95_ms": 86.079}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.707, "count": 14, "max_ms": 97.966, "p95_ms": 97.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.126, "count": 14, "max_ms": 132.352, "p95_ms": 132.352}, "read_interval": {"avg_ms": 2240.977, "count": 14, "max_ms": 16376.982, "p95_ms": 16376.982}, "triton_wait": {"avg_ms": 76.363, "count": 14, "max_ms": 119.267, "p95_ms": 119.267}, "visualize": {"avg_ms": 3.61, "count": 14, "max_ms": 4.099, "p95_ms": 4.099}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.683, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2382}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6051.898, "limit_mb": 8796093022207.996}, "detail": {"pid": 31612, "restart_count": 72, "start_count": 73}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:05:16+09:00", "last_error_summary": "[h264 @ 0x563a468e6cc0] error while decoding MB 79 30, bytestream -5 [in#0/rtsp @ 0x563a45da3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a45da3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:05:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:05:18+09:00", "pid": 31612, "process": {"exists": true, "fd_count": 5, "hwm_mb": 62.582, "name": "ffmpeg", "pid": 31612, "rss_mb": 62.582, "state": "R (running)", "threads": 18}, "restart_count": 72, "restart_count_per_hour": 18, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": true, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.038, "sample_count": 2039, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 285927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:05:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.447, "last_frame_pushed_at": "2026-07-06T22:05:16+09:00", "last_frame_read_age_sec": 4.582, "last_frame_read_at": "2026-07-06T22:05:16+09:00", "mjpeg": {"age_sec": 4.432, "exists": true, "mtime": "2026-07-06T22:05:16.485777+09:00", "mtime_epoch": 1783343116.4857774, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106209}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:05:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1549.414, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.633, "pushed": 0.633, "read": 0.633}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.161, "count": 14, "max_ms": 5.862, "p95_ms": 5.862}, "gstreamer_push": {"avg_ms": 1.273, "count": 14, "max_ms": 1.345, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.996, "count": 14, "max_ms": 83.822, "p95_ms": 83.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.623, "count": 14, "max_ms": 119.252, "p95_ms": 119.252}, "yolo11_640_fire_11class": {"avg_ms": 67.322, "count": 14, "max_ms": 86.079, "p95_ms": 86.079}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.707, "count": 14, "max_ms": 97.966, "p95_ms": 97.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.126, "count": 14, "max_ms": 132.352, "p95_ms": 132.352}, "read_interval": {"avg_ms": 2240.977, "count": 14, "max_ms": 16376.982, "p95_ms": 16376.982}, "triton_wait": {"avg_ms": 76.363, "count": 14, "max_ms": 119.267, "p95_ms": 119.267}, "visualize": {"avg_ms": 3.61, "count": 14, "max_ms": 4.099, "p95_ms": 4.099}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.114, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2391}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6024.371, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e4cac8bfc0] error while decoding MB 103 16, bytestream -5 [in#0/rtsp @ 0x55e4caaffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e4caaffe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:07:53+09:00", "last_error_summary": "[h264 @ 0x55e4cac8bfc0] error while decoding MB 103 16, bytestream -5 [in#0/rtsp @ 0x55e4caaffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e4caaffe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:05:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.067, "last_started_at": "2026-07-06T22:05:18+09:00", "pid": 31612, "process": {"exists": false, "pid": 31612}, "restart_count": 72, "restart_count_per_hour": 18, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.151, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 287025, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:07:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T22:07:53+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T22:07:53+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-06T22:07:53.935172+09:00", "mtime_epoch": 1783343273.9351723, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:07:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1549.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.708, "pushed": 0.708, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.023, "count": 12, "max_ms": 4.167, "p95_ms": 4.167}, "gstreamer_push": {"avg_ms": 1.281, "count": 12, "max_ms": 1.393, "p95_ms": 1.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.229, "count": 12, "max_ms": 79.931, "p95_ms": 79.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.17, "count": 12, "max_ms": 83.553, "p95_ms": 83.553}, "yolo11_640_fire_11class": {"avg_ms": 67.932, "count": 12, "max_ms": 80.575, "p95_ms": 80.575}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.534, "count": 12, "max_ms": 89.833, "p95_ms": 89.833}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.452, "count": 12, "max_ms": 102.376, "p95_ms": 102.376}, "read_interval": {"avg_ms": 2432.786, "count": 13, "max_ms": 15841.744, "p95_ms": 15841.744}, "triton_wait": {"avg_ms": 72.976, "count": 12, "max_ms": 89.845, "p95_ms": 89.845}, "visualize": {"avg_ms": 3.543, "count": 12, "max_ms": 3.711, "p95_ms": 3.711}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.94, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2391}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6023.812, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:07:53+09:00", "last_error_summary": "[h264 @ 0x55e4cac8bfc0] error while decoding MB 103 16, bytestream -5 [in#0/rtsp @ 0x55e4caaffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e4caaffe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:07:53+09:00", "last_error_summary": "[h264 @ 0x55e4cac8bfc0] error while decoding MB 103 16, bytestream -5 [in#0/rtsp @ 0x55e4caaffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e4caaffe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:07:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.126, "last_started_at": "2026-07-06T22:05:18+09:00", "pid": 31612, "process": {"exists": false, "pid": 31612}, "restart_count": 72, "restart_count_per_hour": 18, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.151, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 287026, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:07:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T22:07:54+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-06T22:07:53+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T22:07:53.935172+09:00", "mtime_epoch": 1783343273.9351723, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:07:54+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1549.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.765, "pushed": 0.765, "read": 0.765}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.138, "count": 13, "max_ms": 5.523, "p95_ms": 5.523}, "gstreamer_push": {"avg_ms": 1.289, "count": 13, "max_ms": 1.393, "p95_ms": 1.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.238, "count": 13, "max_ms": 79.931, "p95_ms": 79.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.97, "count": 13, "max_ms": 83.553, "p95_ms": 83.553}, "yolo11_640_fire_11class": {"avg_ms": 68.733, "count": 13, "max_ms": 80.575, "p95_ms": 80.575}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.875, "count": 13, "max_ms": 89.968, "p95_ms": 89.968}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.877, "count": 13, "max_ms": 103.98, "p95_ms": 103.98}, "read_interval": {"avg_ms": 2432.786, "count": 13, "max_ms": 15841.744, "p95_ms": 15841.744}, "triton_wait": {"avg_ms": 74.284, "count": 13, "max_ms": 89.983, "p95_ms": 89.983}, "visualize": {"avg_ms": 3.544, "count": 13, "max_ms": 3.711, "p95_ms": 3.711}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.999, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2391}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6023.816, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:07:53+09:00", "last_error_summary": "[h264 @ 0x55e4cac8bfc0] error while decoding MB 103 16, bytestream -5 [in#0/rtsp @ 0x55e4caaffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e4caaffe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:07:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.136, "last_started_at": "2026-07-06T22:05:18+09:00", "pid": 31612, "process": {"exists": false, "pid": 31612}, "restart_count": 72, "restart_count_per_hour": 18, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.151, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 287026, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:07:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.024, "last_frame_pushed_at": "2026-07-06T22:07:54+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-06T22:07:53+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-06T22:07:53.935172+09:00", "mtime_epoch": 1783343273.9351723, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:07:54+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1549.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.764, "pushed": 0.764, "read": 0.764}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.138, "count": 13, "max_ms": 5.523, "p95_ms": 5.523}, "gstreamer_push": {"avg_ms": 1.289, "count": 13, "max_ms": 1.393, "p95_ms": 1.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.238, "count": 13, "max_ms": 79.931, "p95_ms": 79.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.97, "count": 13, "max_ms": 83.553, "p95_ms": 83.553}, "yolo11_640_fire_11class": {"avg_ms": 68.733, "count": 13, "max_ms": 80.575, "p95_ms": 80.575}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.875, "count": 13, "max_ms": 89.968, "p95_ms": 89.968}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.877, "count": 13, "max_ms": 103.98, "p95_ms": 103.98}, "read_interval": {"avg_ms": 2432.786, "count": 13, "max_ms": 15841.744, "p95_ms": 15841.744}, "triton_wait": {"avg_ms": 74.284, "count": 13, "max_ms": 89.983, "p95_ms": 89.983}, "visualize": {"avg_ms": 3.544, "count": 13, "max_ms": 3.711, "p95_ms": 3.711}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.009, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2391}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6041.594, "limit_mb": 8796093022207.996}, "detail": {"pid": 31764, "restart_count": 73, "start_count": 74}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:07:53+09:00", "last_error_summary": "[h264 @ 0x55e4cac8bfc0] error while decoding MB 103 16, bytestream -5 [in#0/rtsp @ 0x55e4caaffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e4caaffe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:07:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:07:57+09:00", "pid": 31764, "process": {"exists": true, "fd_count": 4, "hwm_mb": 56.957, "name": "ffmpeg", "pid": 31764, "rss_mb": 56.957, "state": "S (sleeping)", "threads": 1}, "restart_count": 73, "restart_count_per_hour": 19, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": true, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.091, "sample_count": 2058, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 287026, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:07:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 5.018, "last_frame_pushed_at": "2026-07-06T22:07:54+09:00", "last_frame_read_age_sec": 5.124, "last_frame_read_at": "2026-07-06T22:07:53+09:00", "mjpeg": {"age_sec": 4.992, "exists": true, "mtime": "2026-07-06T22:07:54.055173+09:00", "mtime_epoch": 1783343274.0551734, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115660}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:07:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1549.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.591, "pushed": 0.591, "read": 0.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.138, "count": 13, "max_ms": 5.523, "p95_ms": 5.523}, "gstreamer_push": {"avg_ms": 1.289, "count": 13, "max_ms": 1.393, "p95_ms": 1.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.238, "count": 13, "max_ms": 79.931, "p95_ms": 79.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.97, "count": 13, "max_ms": 83.553, "p95_ms": 83.553}, "yolo11_640_fire_11class": {"avg_ms": 68.733, "count": 13, "max_ms": 80.575, "p95_ms": 80.575}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.875, "count": 13, "max_ms": 89.968, "p95_ms": 89.968}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.877, "count": 13, "max_ms": 103.98, "p95_ms": 103.98}, "read_interval": {"avg_ms": 2432.786, "count": 13, "max_ms": 15841.744, "p95_ms": 15841.744}, "triton_wait": {"avg_ms": 74.284, "count": 13, "max_ms": 89.983, "p95_ms": 89.983}, "visualize": {"avg_ms": 3.544, "count": 13, "max_ms": 3.711, "p95_ms": 3.711}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.003, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2218}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6084.832, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563f003ab740] error while decoding MB 28 30, bytestream -7 [h264 @ 0x563eff625140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563eff625140] error while decoding MB 0 4, bytestream 7428 [h264 @ 0x563eff639300] left block unavailable for requested intra mode [h264 @ 0x563eff639300] error while decoding MB 0 30, bytestream 1384 [h264 @ 0x563eff72bb00] error while decoding MB 29 2, bytestream 212 [in#0/rtsp @ 0x563eff5f5e00] Error during demuxing: Connection timed out ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 119, "h264_decode_error_count": 389, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:11:40+09:00", "last_error_summary": "[h264 @ 0x563f003ab740] error while decoding MB 28 30, bytestream -7 [h264 @ 0x563eff625140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563eff625140] error while decoding MB 0 4, bytestream 7428 [h264 @ 0x563eff639300] left block unavailable for requested intra mode [h264 @ 0x563eff639300] error while decoding MB 0 30, bytestream 1384 [h264 @ 0x563eff72bb00] error while decoding MB 29 2, bytestream 212 [in#0/rtsp @ 0x563eff5f5e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-06T21:59:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 714.535, "last_started_at": "2026-07-06T21:59:45+09:00", "pid": 31353, "process": {"exists": false, "pid": 31353}, "restart_count": 128, "restart_count_per_hour": 6, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 129}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.403, "sample_count": 2601, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 266229, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:11:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T22:11:40+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-06T22:11:40+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T22:11:40.097382+09:00", "mtime_epoch": 1783343500.0973823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118435}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:11:40+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1665.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.305, "pushed": 3.305, "read": 3.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.547, "count": 99, "max_ms": 5.459, "p95_ms": 4.301}, "gstreamer_push": {"avg_ms": 1.299, "count": 99, "max_ms": 1.447, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.704, "count": 99, "max_ms": 86.917, "p95_ms": 81.587}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.293, "count": 99, "max_ms": 109.229, "p95_ms": 104.571}, "yolo11_640_fire_11class": {"avg_ms": 75.498, "count": 99, "max_ms": 88.157, "p95_ms": 84.523}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.485, "count": 99, "max_ms": 100.034, "p95_ms": 94.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.277, "count": 99, "max_ms": 120.191, "p95_ms": 116.591}, "read_interval": {"avg_ms": 303.06, "count": 99, "max_ms": 19119.398, "p95_ms": 130.391}, "triton_wait": {"avg_ms": 86.139, "count": 99, "max_ms": 109.239, "p95_ms": 104.584}, "visualize": {"avg_ms": 3.607, "count": 99, "max_ms": 4.451, "p95_ms": 3.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2218}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6084.141, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:11:40+09:00", "last_error_summary": "[h264 @ 0x563f003ab740] error while decoding MB 28 30, bytestream -7 [h264 @ 0x563eff625140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563eff625140] error while decoding MB 0 4, bytestream 7428 [h264 @ 0x563eff639300] left block unavailable for requested intra mode [h264 @ 0x563eff639300] error while decoding MB 0 30, bytestream 1384 [h264 @ 0x563eff72bb00] error while decoding MB 29 2, bytestream 212 [in#0/rtsp @ 0x563eff5f5e00] Error during demuxing: Connection timed out ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 119, "h264_decode_error_count": 389, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:11:40+09:00", "last_error_summary": "[h264 @ 0x563f003ab740] error while decoding MB 28 30, bytestream -7 [h264 @ 0x563eff625140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563eff625140] error while decoding MB 0 4, bytestream 7428 [h264 @ 0x563eff639300] left block unavailable for requested intra mode [h264 @ 0x563eff639300] error while decoding MB 0 30, bytestream 1384 [h264 @ 0x563eff72bb00] error while decoding MB 29 2, bytestream 212 [in#0/rtsp @ 0x563eff5f5e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-06T22:11:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 714.588, "last_started_at": "2026-07-06T21:59:45+09:00", "pid": 31353, "process": {"exists": false, "pid": 31353}, "restart_count": 128, "restart_count_per_hour": 6, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 129}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.403, "sample_count": 2602, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 266230, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:11:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T22:11:40+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T22:11:40+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T22:11:40.097382+09:00", "mtime_epoch": 1783343500.0973823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118435}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:11:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1665.961, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.547, "count": 99, "max_ms": 5.459, "p95_ms": 4.301}, "gstreamer_push": {"avg_ms": 1.298, "count": 99, "max_ms": 1.447, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.707, "count": 99, "max_ms": 86.917, "p95_ms": 81.587}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.287, "count": 99, "max_ms": 109.229, "p95_ms": 104.571}, "yolo11_640_fire_11class": {"avg_ms": 75.514, "count": 99, "max_ms": 88.157, "p95_ms": 84.523}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.409, "count": 99, "max_ms": 100.034, "p95_ms": 93.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.198, "count": 99, "max_ms": 120.191, "p95_ms": 116.591}, "read_interval": {"avg_ms": 303.06, "count": 99, "max_ms": 19119.398, "p95_ms": 130.391}, "triton_wait": {"avg_ms": 86.063, "count": 99, "max_ms": 109.239, "p95_ms": 104.584}, "visualize": {"avg_ms": 3.606, "count": 99, "max_ms": 4.451, "p95_ms": 3.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2218}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6084.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 119, "h264_decode_error_count": 389, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:11:40+09:00", "last_error_summary": "[h264 @ 0x563f003ab740] error while decoding MB 28 30, bytestream -7 [h264 @ 0x563eff625140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563eff625140] error while decoding MB 0 4, bytestream 7428 [h264 @ 0x563eff639300] left block unavailable for requested intra mode [h264 @ 0x563eff639300] error while decoding MB 0 30, bytestream 1384 [h264 @ 0x563eff72bb00] error while decoding MB 29 2, bytestream 212 [in#0/rtsp @ 0x563eff5f5e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-06T22:11:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 714.595, "last_started_at": "2026-07-06T21:59:45+09:00", "pid": 31353, "process": {"exists": false, "pid": 31353}, "restart_count": 128, "restart_count_per_hour": 6, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 129}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.403, "sample_count": 2602, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 266230, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:11:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-06T22:11:40+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-06T22:11:40+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-06T22:11:40.097382+09:00", "mtime_epoch": 1783343500.0973823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118435}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:11:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1665.961, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.31, "pushed": 3.31, "read": 3.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.547, "count": 99, "max_ms": 5.459, "p95_ms": 4.301}, "gstreamer_push": {"avg_ms": 1.298, "count": 99, "max_ms": 1.447, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.707, "count": 99, "max_ms": 86.917, "p95_ms": 81.587}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.287, "count": 99, "max_ms": 109.229, "p95_ms": 104.571}, "yolo11_640_fire_11class": {"avg_ms": 75.514, "count": 99, "max_ms": 88.157, "p95_ms": 84.523}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.409, "count": 99, "max_ms": 100.034, "p95_ms": 93.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.198, "count": 99, "max_ms": 120.191, "p95_ms": 116.591}, "read_interval": {"avg_ms": 304.931, "count": 98, "max_ms": 19119.398, "p95_ms": 130.391}, "triton_wait": {"avg_ms": 86.063, "count": 99, "max_ms": 109.239, "p95_ms": 104.584}, "visualize": {"avg_ms": 3.606, "count": 99, "max_ms": 4.451, "p95_ms": 3.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2218}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6194.586, "limit_mb": 8796093022207.996}, "detail": {"pid": 31916, "restart_count": 129, "start_count": 130}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 119, "h264_decode_error_count": 389, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:11:40+09:00", "last_error_summary": "[h264 @ 0x563f003ab740] error while decoding MB 28 30, bytestream -7 [h264 @ 0x563eff625140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563eff625140] error while decoding MB 0 4, bytestream 7428 [h264 @ 0x563eff639300] left block unavailable for requested intra mode [h264 @ 0x563eff639300] error while decoding MB 0 30, bytestream 1384 [h264 @ 0x563eff72bb00] error while decoding MB 29 2, bytestream 212 [in#0/rtsp @ 0x563eff5f5e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-06T22:11:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:11:43+09:00", "pid": 31916, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.309, "name": "ffmpeg", "pid": 31916, "rss_mb": 139.309, "state": "S (sleeping)", "threads": 106}, "restart_count": 129, "restart_count_per_hour": 7, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 130}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.403, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 266230, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:11:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.814, "last_frame_pushed_at": "2026-07-06T22:11:40+09:00", "last_frame_read_age_sec": 4.915, "last_frame_read_at": "2026-07-06T22:11:40+09:00", "mjpeg": {"age_sec": 4.798, "exists": true, "mtime": "2026-07-06T22:11:40.205384+09:00", "mtime_epoch": 1783343500.2053835, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116252}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:11:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1665.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.871, "pushed": 1.871, "read": 1.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.525, "count": 56, "max_ms": 4.911, "p95_ms": 4.118}, "gstreamer_push": {"avg_ms": 1.306, "count": 56, "max_ms": 1.447, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.036, "count": 56, "max_ms": 86.917, "p95_ms": 80.703}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.275, "count": 56, "max_ms": 107.933, "p95_ms": 102.336}, "yolo11_640_fire_11class": {"avg_ms": 76.228, "count": 56, "max_ms": 88.157, "p95_ms": 83.981}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.177, "count": 56, "max_ms": 100.034, "p95_ms": 95.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.655, "count": 56, "max_ms": 120.191, "p95_ms": 114.794}, "read_interval": {"avg_ms": 456.693, "count": 55, "max_ms": 19119.398, "p95_ms": 131.541}, "triton_wait": {"avg_ms": 86.572, "count": 56, "max_ms": 107.946, "p95_ms": 102.349}, "visualize": {"avg_ms": 3.584, "count": 56, "max_ms": 4.106, "p95_ms": 3.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2427}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6023.859, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5555f7934280] error while decoding MB 106 31, bytestream -5 [in#0/rtsp @ 0x5555f6d16e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5555f6d16e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:16:08+09:00", "last_error_summary": "[h264 @ 0x5555f7934280] error while decoding MB 106 31, bytestream -5 [in#0/rtsp @ 0x5555f6d16e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5555f6d16e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:07:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 491.716, "last_started_at": "2026-07-06T22:07:57+09:00", "pid": 31764, "process": {"exists": false, "pid": 31764}, "restart_count": 73, "restart_count_per_hour": 17, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.602, "sample_count": 2580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 291293, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:16:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-06T22:16:08+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-06T22:16:08+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-06T22:16:08.704689+09:00", "mtime_epoch": 1783343768.7046888, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 113981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:16:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1551.734, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.303, "pushed": 3.303, "read": 3.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.167, "count": 99, "max_ms": 6.296, "p95_ms": 5.462}, "gstreamer_push": {"avg_ms": 1.284, "count": 99, "max_ms": 1.547, "p95_ms": 1.418}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.557, "count": 99, "max_ms": 86.86, "p95_ms": 81.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.811, "count": 99, "max_ms": 120.442, "p95_ms": 112.157}, "yolo11_640_fire_11class": {"avg_ms": 74.32, "count": 99, "max_ms": 87.354, "p95_ms": 84.489}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.653, "count": 99, "max_ms": 99.99, "p95_ms": 95.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.219, "count": 99, "max_ms": 133.594, "p95_ms": 124.842}, "read_interval": {"avg_ms": 303.049, "count": 99, "max_ms": 19067.951, "p95_ms": 138.278}, "triton_wait": {"avg_ms": 85.369, "count": 99, "max_ms": 120.456, "p95_ms": 112.171}, "visualize": {"avg_ms": 3.643, "count": 99, "max_ms": 4.694, "p95_ms": 4.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2427}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6023.105, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:16:08+09:00", "last_error_summary": "[h264 @ 0x5555f7934280] error while decoding MB 106 31, bytestream -5 [in#0/rtsp @ 0x5555f6d16e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5555f6d16e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:16:08+09:00", "last_error_summary": "[h264 @ 0x5555f7934280] error while decoding MB 106 31, bytestream -5 [in#0/rtsp @ 0x5555f6d16e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5555f6d16e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:16:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 491.753, "last_started_at": "2026-07-06T22:07:57+09:00", "pid": 31764, "process": {"exists": false, "pid": 31764}, "restart_count": 73, "restart_count_per_hour": 17, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.602, "sample_count": 2581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 291294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:16:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:16:08+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-06T22:16:08+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-06T22:16:08.704689+09:00", "mtime_epoch": 1783343768.7046888, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 113981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:16:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1551.734, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.167, "count": 99, "max_ms": 6.296, "p95_ms": 5.462}, "gstreamer_push": {"avg_ms": 1.284, "count": 99, "max_ms": 1.547, "p95_ms": 1.418}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.375, "count": 99, "max_ms": 86.86, "p95_ms": 81.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.839, "count": 99, "max_ms": 120.442, "p95_ms": 112.157}, "yolo11_640_fire_11class": {"avg_ms": 74.342, "count": 99, "max_ms": 87.354, "p95_ms": 84.489}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.633, "count": 99, "max_ms": 99.99, "p95_ms": 95.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.197, "count": 99, "max_ms": 133.594, "p95_ms": 124.842}, "read_interval": {"avg_ms": 303.049, "count": 99, "max_ms": 19067.951, "p95_ms": 138.278}, "triton_wait": {"avg_ms": 85.349, "count": 99, "max_ms": 120.456, "p95_ms": 112.171}, "visualize": {"avg_ms": 3.644, "count": 99, "max_ms": 4.694, "p95_ms": 4.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2427}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6023.164, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:16:08+09:00", "last_error_summary": "[h264 @ 0x5555f7934280] error while decoding MB 106 31, bytestream -5 [in#0/rtsp @ 0x5555f6d16e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5555f6d16e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:16:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 491.76, "last_started_at": "2026-07-06T22:07:57+09:00", "pid": 31764, "process": {"exists": false, "pid": 31764}, "restart_count": 73, "restart_count_per_hour": 17, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.602, "sample_count": 2581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 291294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:16:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T22:16:08+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-06T22:16:08+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T22:16:08.704689+09:00", "mtime_epoch": 1783343768.7046888, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 113981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:16:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1551.734, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.311, "pushed": 3.311, "read": 3.278}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.167, "count": 99, "max_ms": 6.296, "p95_ms": 5.462}, "gstreamer_push": {"avg_ms": 1.284, "count": 99, "max_ms": 1.547, "p95_ms": 1.418}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.375, "count": 99, "max_ms": 86.86, "p95_ms": 81.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.839, "count": 99, "max_ms": 120.442, "p95_ms": 112.157}, "yolo11_640_fire_11class": {"avg_ms": 74.342, "count": 99, "max_ms": 87.354, "p95_ms": 84.489}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.633, "count": 99, "max_ms": 99.99, "p95_ms": 95.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.197, "count": 99, "max_ms": 133.594, "p95_ms": 124.842}, "read_interval": {"avg_ms": 305.012, "count": 98, "max_ms": 19067.951, "p95_ms": 138.278}, "triton_wait": {"avg_ms": 85.349, "count": 99, "max_ms": 120.456, "p95_ms": 112.171}, "visualize": {"avg_ms": 3.644, "count": 99, "max_ms": 4.694, "p95_ms": 4.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2427}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6038.051, "limit_mb": 8796093022207.996}, "detail": {"pid": 32023, "restart_count": 74, "start_count": 75}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:16:08+09:00", "last_error_summary": "[h264 @ 0x5555f7934280] error while decoding MB 106 31, bytestream -5 [in#0/rtsp @ 0x5555f6d16e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5555f6d16e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:16:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:16:10+09:00", "pid": 32023, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.055, "name": "ffmpeg", "pid": 32023, "rss_mb": 58.055, "state": "S (sleeping)", "threads": 1}, "restart_count": 74, "restart_count_per_hour": 18, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": true, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.574, "sample_count": 2544, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 291294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:16:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.085, "last_frame_pushed_at": "2026-07-06T22:16:08+09:00", "last_frame_read_age_sec": 4.182, "last_frame_read_at": "2026-07-06T22:16:08+09:00", "mjpeg": {"age_sec": 4.069, "exists": true, "mtime": "2026-07-06T22:16:08.808690+09:00", "mtime_epoch": 1783343768.8086903, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 113602}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:16:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1551.734, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.136, "pushed": 2.136, "read": 2.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.097, "count": 64, "max_ms": 6.072, "p95_ms": 4.841}, "gstreamer_push": {"avg_ms": 1.271, "count": 64, "max_ms": 1.547, "p95_ms": 1.418}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.029, "count": 64, "max_ms": 84.611, "p95_ms": 81.204}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.202, "count": 64, "max_ms": 116.062, "p95_ms": 110.796}, "yolo11_640_fire_11class": {"avg_ms": 73.118, "count": 64, "max_ms": 85.498, "p95_ms": 83.677}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.795, "count": 64, "max_ms": 99.99, "p95_ms": 93.707}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.499, "count": 64, "max_ms": 130.237, "p95_ms": 123.778}, "read_interval": {"avg_ms": 410.684, "count": 63, "max_ms": 19067.951, "p95_ms": 137.652}, "triton_wait": {"avg_ms": 83.781, "count": 64, "max_ms": 116.075, "p95_ms": 110.808}, "visualize": {"avg_ms": 3.617, "count": 64, "max_ms": 4.694, "p95_ms": 4.424}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6035.898, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562a4993ba40] error while decoding MB 61 17, bytestream -5 [in#0/rtsp @ 0x562a48d20e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562a48d20e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:18:22+09:00", "last_error_summary": "[h264 @ 0x562a4993ba40] error while decoding MB 61 17, bytestream -5 [in#0/rtsp @ 0x562a48d20e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562a48d20e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:16:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.998, "last_started_at": "2026-07-06T22:16:10+09:00", "pid": 32023, "process": {"exists": false, "pid": 32023}, "restart_count": 74, "restart_count_per_hour": 18, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 34, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.542, "sample_count": 2193, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292165, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:18:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T22:18:22+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T22:18:22+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-06T22:18:22.818523+09:00", "mtime_epoch": 1783343902.8185227, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104252}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:18:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1552.25, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.728, "pushed": 0.728, "read": 0.784}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.152, "count": 13, "max_ms": 4.824, "p95_ms": 4.824}, "gstreamer_push": {"avg_ms": 1.254, "count": 13, "max_ms": 1.357, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.481, "count": 13, "max_ms": 83.007, "p95_ms": 83.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.716, "count": 13, "max_ms": 103.628, "p95_ms": 103.628}, "yolo11_640_fire_11class": {"avg_ms": 67.474, "count": 13, "max_ms": 78.854, "p95_ms": 78.854}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.98, "count": 13, "max_ms": 91.671, "p95_ms": 91.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.2, "count": 13, "max_ms": 116.542, "p95_ms": 116.542}, "read_interval": {"avg_ms": 2394.091, "count": 14, "max_ms": 16603.437, "p95_ms": 16603.437}, "triton_wait": {"avg_ms": 75.519, "count": 13, "max_ms": 103.641, "p95_ms": 103.641}, "visualize": {"avg_ms": 3.594, "count": 13, "max_ms": 3.837, "p95_ms": 3.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.85, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6035.422, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:18:22+09:00", "last_error_summary": "[h264 @ 0x562a4993ba40] error while decoding MB 61 17, bytestream -5 [in#0/rtsp @ 0x562a48d20e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562a48d20e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:18:22+09:00", "last_error_summary": "[h264 @ 0x562a4993ba40] error while decoding MB 61 17, bytestream -5 [in#0/rtsp @ 0x562a48d20e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562a48d20e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:18:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.038, "last_started_at": "2026-07-06T22:16:10+09:00", "pid": 32023, "process": {"exists": false, "pid": 32023}, "restart_count": 74, "restart_count_per_hour": 18, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 34, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.182, "sample_count": 2193, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:18:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T22:18:22+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-06T22:18:22+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-06T22:18:22.818523+09:00", "mtime_epoch": 1783343902.8185227, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104252}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:18:22+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1552.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.783, "pushed": 0.783, "read": 0.783}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.14, "count": 14, "max_ms": 4.824, "p95_ms": 4.824}, "gstreamer_push": {"avg_ms": 1.252, "count": 14, "max_ms": 1.357, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.507, "count": 14, "max_ms": 83.007, "p95_ms": 83.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.293, "count": 14, "max_ms": 103.628, "p95_ms": 103.628}, "yolo11_640_fire_11class": {"avg_ms": 67.915, "count": 14, "max_ms": 78.854, "p95_ms": 78.854}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.541, "count": 14, "max_ms": 91.671, "p95_ms": 91.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.622, "count": 14, "max_ms": 116.542, "p95_ms": 116.542}, "read_interval": {"avg_ms": 2394.091, "count": 14, "max_ms": 16603.437, "p95_ms": 16603.437}, "triton_wait": {"avg_ms": 75.97, "count": 14, "max_ms": 103.641, "p95_ms": 103.641}, "visualize": {"avg_ms": 3.593, "count": 14, "max_ms": 3.837, "p95_ms": 3.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.889, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6035.48, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:18:22+09:00", "last_error_summary": "[h264 @ 0x562a4993ba40] error while decoding MB 61 17, bytestream -5 [in#0/rtsp @ 0x562a48d20e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562a48d20e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:18:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.045, "last_started_at": "2026-07-06T22:16:10+09:00", "pid": 32023, "process": {"exists": false, "pid": 32023}, "restart_count": 74, "restart_count_per_hour": 18, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 34, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.182, "sample_count": 2193, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:18:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T22:18:22+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T22:18:22+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-06T22:18:22.818523+09:00", "mtime_epoch": 1783343902.8185227, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104252}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:18:22+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1552.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.14, "count": 14, "max_ms": 4.824, "p95_ms": 4.824}, "gstreamer_push": {"avg_ms": 1.252, "count": 14, "max_ms": 1.357, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.507, "count": 14, "max_ms": 83.007, "p95_ms": 83.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.293, "count": 14, "max_ms": 103.628, "p95_ms": 103.628}, "yolo11_640_fire_11class": {"avg_ms": 67.915, "count": 14, "max_ms": 78.854, "p95_ms": 78.854}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.541, "count": 14, "max_ms": 91.671, "p95_ms": 91.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.622, "count": 14, "max_ms": 116.542, "p95_ms": 116.542}, "read_interval": {"avg_ms": 2394.091, "count": 14, "max_ms": 16603.437, "p95_ms": 16603.437}, "triton_wait": {"avg_ms": 75.97, "count": 14, "max_ms": 103.641, "p95_ms": 103.641}, "visualize": {"avg_ms": 3.593, "count": 14, "max_ms": 3.837, "p95_ms": 3.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.896, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6058.441, "limit_mb": 8796093022207.996}, "detail": {"pid": 32175, "restart_count": 75, "start_count": 76}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:18:22+09:00", "last_error_summary": "[h264 @ 0x562a4993ba40] error while decoding MB 61 17, bytestream -5 [in#0/rtsp @ 0x562a48d20e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562a48d20e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:18:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:18:25+09:00", "pid": 32175, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.293, "name": "ffmpeg", "pid": 32175, "rss_mb": 58.293, "state": "S (sleeping)", "threads": 1}, "restart_count": 75, "restart_count_per_hour": 19, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 34, "rtsp_tolerant_mode": false, "running": true, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.542, "sample_count": 2150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:18:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.829, "last_frame_pushed_at": "2026-07-06T22:18:22+09:00", "last_frame_read_age_sec": 4.925, "last_frame_read_at": "2026-07-06T22:18:22+09:00", "mjpeg": {"age_sec": 4.812, "exists": true, "mtime": "2026-07-06T22:18:22.926524+09:00", "mtime_epoch": 1783343902.9265242, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:18:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1552.25, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.616, "pushed": 0.616, "read": 0.616}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.14, "count": 14, "max_ms": 4.824, "p95_ms": 4.824}, "gstreamer_push": {"avg_ms": 1.252, "count": 14, "max_ms": 1.357, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.507, "count": 14, "max_ms": 83.007, "p95_ms": 83.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.293, "count": 14, "max_ms": 103.628, "p95_ms": 103.628}, "yolo11_640_fire_11class": {"avg_ms": 67.915, "count": 14, "max_ms": 78.854, "p95_ms": 78.854}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.541, "count": 14, "max_ms": 91.671, "p95_ms": 91.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.622, "count": 14, "max_ms": 116.542, "p95_ms": 116.542}, "read_interval": {"avg_ms": 2394.091, "count": 14, "max_ms": 16603.437, "p95_ms": 16603.437}, "triton_wait": {"avg_ms": 75.97, "count": 14, "max_ms": 103.641, "p95_ms": 103.641}, "visualize": {"avg_ms": 3.593, "count": 14, "max_ms": 3.837, "p95_ms": 3.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.713, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2269}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6092.691, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5612d850dc40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612d850dc40] error while decoding MB 0 4, bytestream 2606 [h264 @ 0x5612d8fd2200] left block unavailable for requested intra mode [h264 @ 0x5612d8fd2200] error while decoding MB 0 9, bytestream 5769 [h264 @ 0x5612d85acc80] cabac decode of qscale diff failed at 49 19 [h264 @ 0x5612d85acc80] error while decoding MB 49 19, bytestream 4328 [h264 @ 0x5612d850dc40] left block unavailable for requested intra mode [", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 120, "h264_decode_error_count": 398, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:23:10+09:00", "last_error_summary": "[h264 @ 0x5612d850dc40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612d850dc40] error while decoding MB 0 4, bytestream 2606 [h264 @ 0x5612d8fd2200] left block unavailable for requested intra mode [h264 @ 0x5612d8fd2200] error while decoding MB 0 9, bytestream 5769 [h264 @ 0x5612d85acc80] cabac decode of qscale diff failed at 49 19 [h264 @ 0x5612d85acc80] error while decoding MB 49 19, bytestream 4328 [h264 @ 0x5612d850dc40] left block unavailable for requested intra mode [", "last_exit_at": "2026-07-06T22:11:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 687.791, "last_started_at": "2026-07-06T22:11:43+09:00", "pid": 31916, "process": {"exists": false, "pid": 31916}, "restart_count": 129, "restart_count_per_hour": 6, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 130}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.534, "sample_count": 2553, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 272348, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:23:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T22:23:10+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T22:23:10+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-06T22:23:10.738722+09:00", "mtime_epoch": 1783344190.738722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123110}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:23:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1669.055, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.304, "pushed": 3.304, "read": 3.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.74, "count": 99, "max_ms": 5.27, "p95_ms": 5.056}, "gstreamer_push": {"avg_ms": 1.331, "count": 99, "max_ms": 1.534, "p95_ms": 1.466}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.158, "count": 99, "max_ms": 89.992, "p95_ms": 84.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.025, "count": 99, "max_ms": 116.07, "p95_ms": 108.052}, "yolo11_640_fire_11class": {"avg_ms": 76.192, "count": 99, "max_ms": 101.75, "p95_ms": 87.558}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.648, "count": 99, "max_ms": 100.154, "p95_ms": 96.121}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.036, "count": 99, "max_ms": 128.228, "p95_ms": 120.496}, "read_interval": {"avg_ms": 303.126, "count": 99, "max_ms": 19100.971, "p95_ms": 137.879}, "triton_wait": {"avg_ms": 85.576, "count": 99, "max_ms": 116.082, "p95_ms": 108.064}, "visualize": {"avg_ms": 3.684, "count": 99, "max_ms": 4.498, "p95_ms": 4.172}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2269}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6092.043, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:23:10+09:00", "last_error_summary": "[h264 @ 0x5612d850dc40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612d850dc40] error while decoding MB 0 4, bytestream 2606 [h264 @ 0x5612d8fd2200] left block unavailable for requested intra mode [h264 @ 0x5612d8fd2200] error while decoding MB 0 9, bytestream 5769 [h264 @ 0x5612d85acc80] cabac decode of qscale diff failed at 49 19 [h264 @ 0x5612d85acc80] error while decoding MB 49 19, bytestream 4328 [h264 @ 0x5612d850dc40] left block unavailable for requested intra mode [", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 120, "h264_decode_error_count": 398, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:23:10+09:00", "last_error_summary": "[h264 @ 0x5612d850dc40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612d850dc40] error while decoding MB 0 4, bytestream 2606 [h264 @ 0x5612d8fd2200] left block unavailable for requested intra mode [h264 @ 0x5612d8fd2200] error while decoding MB 0 9, bytestream 5769 [h264 @ 0x5612d85acc80] cabac decode of qscale diff failed at 49 19 [h264 @ 0x5612d85acc80] error while decoding MB 49 19, bytestream 4328 [h264 @ 0x5612d850dc40] left block unavailable for requested intra mode [", "last_exit_at": "2026-07-06T22:23:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 687.865, "last_started_at": "2026-07-06T22:11:43+09:00", "pid": 31916, "process": {"exists": false, "pid": 31916}, "restart_count": 129, "restart_count_per_hour": 6, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 130}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.726, "sample_count": 2553, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 272349, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:23:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:23:10+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-06T22:23:10+09:00", "mjpeg": {"age_sec": 0.129, "exists": true, "mtime": "2026-07-06T22:23:10.738722+09:00", "mtime_epoch": 1783344190.738722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123110}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:23:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1669.055, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.308, "pushed": 3.308, "read": 3.275}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.733, "count": 99, "max_ms": 5.27, "p95_ms": 5.056}, "gstreamer_push": {"avg_ms": 1.331, "count": 99, "max_ms": 1.534, "p95_ms": 1.466}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.101, "count": 99, "max_ms": 89.992, "p95_ms": 84.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.35, "count": 99, "max_ms": 116.07, "p95_ms": 108.373}, "yolo11_640_fire_11class": {"avg_ms": 76.284, "count": 99, "max_ms": 101.75, "p95_ms": 87.558}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.643, "count": 99, "max_ms": 100.154, "p95_ms": 96.121}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.275, "count": 99, "max_ms": 128.228, "p95_ms": 120.606}, "read_interval": {"avg_ms": 305.114, "count": 98, "max_ms": 19100.971, "p95_ms": 137.879}, "triton_wait": {"avg_ms": 85.818, "count": 99, "max_ms": 116.082, "p95_ms": 108.386}, "visualize": {"avg_ms": 3.685, "count": 99, "max_ms": 4.498, "p95_ms": 4.172}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2269}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6092.113, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 120, "h264_decode_error_count": 398, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:23:10+09:00", "last_error_summary": "[h264 @ 0x5612d850dc40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612d850dc40] error while decoding MB 0 4, bytestream 2606 [h264 @ 0x5612d8fd2200] left block unavailable for requested intra mode [h264 @ 0x5612d8fd2200] error while decoding MB 0 9, bytestream 5769 [h264 @ 0x5612d85acc80] cabac decode of qscale diff failed at 49 19 [h264 @ 0x5612d85acc80] error while decoding MB 49 19, bytestream 4328 [h264 @ 0x5612d850dc40] left block unavailable for requested intra mode [", "last_exit_at": "2026-07-06T22:23:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 687.873, "last_started_at": "2026-07-06T22:11:43+09:00", "pid": 31916, "process": {"exists": false, "pid": 31916}, "restart_count": 129, "restart_count_per_hour": 6, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 130}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.726, "sample_count": 2553, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 272349, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:23:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T22:23:10+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-06T22:23:10+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-06T22:23:10.738722+09:00", "mtime_epoch": 1783344190.738722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123110}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:23:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1669.055, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.307, "pushed": 3.307, "read": 3.274}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.733, "count": 99, "max_ms": 5.27, "p95_ms": 5.056}, "gstreamer_push": {"avg_ms": 1.331, "count": 99, "max_ms": 1.534, "p95_ms": 1.466}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.101, "count": 99, "max_ms": 89.992, "p95_ms": 84.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.35, "count": 99, "max_ms": 116.07, "p95_ms": 108.373}, "yolo11_640_fire_11class": {"avg_ms": 76.284, "count": 99, "max_ms": 101.75, "p95_ms": 87.558}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.643, "count": 99, "max_ms": 100.154, "p95_ms": 96.121}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.275, "count": 99, "max_ms": 128.228, "p95_ms": 120.606}, "read_interval": {"avg_ms": 305.114, "count": 98, "max_ms": 19100.971, "p95_ms": 137.879}, "triton_wait": {"avg_ms": 85.818, "count": 99, "max_ms": 116.082, "p95_ms": 108.386}, "visualize": {"avg_ms": 3.685, "count": 99, "max_ms": 4.498, "p95_ms": 4.172}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2269}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6197.047, "limit_mb": 8796093022207.996}, "detail": {"pid": 32327, "restart_count": 130, "start_count": 131}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 120, "h264_decode_error_count": 398, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:23:10+09:00", "last_error_summary": "[h264 @ 0x5612d850dc40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5612d850dc40] error while decoding MB 0 4, bytestream 2606 [h264 @ 0x5612d8fd2200] left block unavailable for requested intra mode [h264 @ 0x5612d8fd2200] error while decoding MB 0 9, bytestream 5769 [h264 @ 0x5612d85acc80] cabac decode of qscale diff failed at 49 19 [h264 @ 0x5612d85acc80] error while decoding MB 49 19, bytestream 4328 [h264 @ 0x5612d850dc40] left block unavailable for requested intra mode [", "last_exit_at": "2026-07-06T22:23:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:23:13+09:00", "pid": 32327, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.266, "name": "ffmpeg", "pid": 32327, "rss_mb": 139.266, "state": "S (sleeping)", "threads": 106}, "restart_count": 130, "restart_count_per_hour": 7, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 131}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.962, "sample_count": 2503, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 272349, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:23:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.363, "last_frame_pushed_at": "2026-07-06T22:23:10+09:00", "last_frame_read_age_sec": 4.486, "last_frame_read_at": "2026-07-06T22:23:10+09:00", "mjpeg": {"age_sec": 4.346, "exists": true, "mtime": "2026-07-06T22:23:10.878724+09:00", "mtime_epoch": 1783344190.878724, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:23:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1669.055, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.004, "pushed": 2.004, "read": 1.971}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.683, "count": 60, "max_ms": 5.085, "p95_ms": 4.621}, "gstreamer_push": {"avg_ms": 1.327, "count": 60, "max_ms": 1.479, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.15, "count": 60, "max_ms": 89.992, "p95_ms": 84.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.429, "count": 60, "max_ms": 116.07, "p95_ms": 103.338}, "yolo11_640_fire_11class": {"avg_ms": 76.968, "count": 60, "max_ms": 101.75, "p95_ms": 85.466}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.104, "count": 60, "max_ms": 100.154, "p95_ms": 96.121}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.363, "count": 60, "max_ms": 128.228, "p95_ms": 115.726}, "read_interval": {"avg_ms": 432.992, "count": 59, "max_ms": 19100.971, "p95_ms": 133.037}, "triton_wait": {"avg_ms": 85.957, "count": 60, "max_ms": 116.082, "p95_ms": 103.35}, "visualize": {"avg_ms": 3.693, "count": 60, "max_ms": 4.498, "p95_ms": 3.991}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2458}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6025.613, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556d192ae680] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x556d18b66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d18b66e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:24:07+09:00", "last_error_summary": "[h264 @ 0x556d192ae680] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x556d18b66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d18b66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:18:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 341.956, "last_started_at": "2026-07-06T22:18:25+09:00", "pid": 32175, "process": {"exists": false, "pid": 32175}, "restart_count": 75, "restart_count_per_hour": 18, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 34, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.623, "sample_count": 2480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295004, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:24:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-06T22:24:07+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T22:24:07+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-06T22:24:07.635583+09:00", "mtime_epoch": 1783344247.6355827, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97896}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:24:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1553.539, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.697, "pushed": 0.697, "read": 0.755}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.088, "count": 12, "max_ms": 4.542, "p95_ms": 4.542}, "gstreamer_push": {"avg_ms": 1.287, "count": 12, "max_ms": 1.424, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.329, "count": 12, "max_ms": 83.959, "p95_ms": 83.959}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.628, "count": 12, "max_ms": 88.02, "p95_ms": 88.02}, "yolo11_640_fire_11class": {"avg_ms": 71.194, "count": 12, "max_ms": 89.791, "p95_ms": 89.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.502, "count": 12, "max_ms": 96.962, "p95_ms": 96.962}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.615, "count": 12, "max_ms": 109.882, "p95_ms": 109.882}, "read_interval": {"avg_ms": 2387.865, "count": 13, "max_ms": 16011.192, "p95_ms": 16011.192}, "triton_wait": {"avg_ms": 78.916, "count": 12, "max_ms": 96.974, "p95_ms": 96.974}, "visualize": {"avg_ms": 3.577, "count": 12, "max_ms": 3.68, "p95_ms": 3.68}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.215, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2458}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6024.852, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:24:07+09:00", "last_error_summary": "[h264 @ 0x556d192ae680] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x556d18b66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d18b66e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:24:07+09:00", "last_error_summary": "[h264 @ 0x556d192ae680] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x556d18b66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d18b66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:24:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 342.021, "last_started_at": "2026-07-06T22:18:25+09:00", "pid": 32175, "process": {"exists": false, "pid": 32175}, "restart_count": 75, "restart_count_per_hour": 18, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 34, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.691, "sample_count": 2480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295005, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:24:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T22:24:07+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-06T22:24:07+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-06T22:24:07.635583+09:00", "mtime_epoch": 1783344247.6355827, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97896}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:24:07+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1553.539, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.752, "pushed": 0.752, "read": 0.752}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.093, "count": 13, "max_ms": 4.542, "p95_ms": 4.542}, "gstreamer_push": {"avg_ms": 1.283, "count": 13, "max_ms": 1.424, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.318, "count": 13, "max_ms": 83.959, "p95_ms": 83.959}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.085, "count": 13, "max_ms": 107.568, "p95_ms": 107.568}, "yolo11_640_fire_11class": {"avg_ms": 71.842, "count": 13, "max_ms": 89.791, "p95_ms": 89.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.192, "count": 13, "max_ms": 96.962, "p95_ms": 96.962}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.83, "count": 13, "max_ms": 120.407, "p95_ms": 120.407}, "read_interval": {"avg_ms": 2387.865, "count": 13, "max_ms": 16011.192, "p95_ms": 16011.192}, "triton_wait": {"avg_ms": 81.122, "count": 13, "max_ms": 107.582, "p95_ms": 107.582}, "visualize": {"avg_ms": 3.583, "count": 13, "max_ms": 3.68, "p95_ms": 3.68}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.28, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2458}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6024.844, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:24:07+09:00", "last_error_summary": "[h264 @ 0x556d192ae680] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x556d18b66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d18b66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:24:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 342.028, "last_started_at": "2026-07-06T22:18:25+09:00", "pid": 32175, "process": {"exists": false, "pid": 32175}, "restart_count": 75, "restart_count_per_hour": 18, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 34, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.691, "sample_count": 2480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295005, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:24:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T22:24:07+09:00", "last_frame_read_age_sec": 0.135, "last_frame_read_at": "2026-07-06T22:24:07+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-06T22:24:07.635583+09:00", "mtime_epoch": 1783344247.6355827, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97896}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:24:07+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1553.539, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.752, "pushed": 0.752, "read": 0.752}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.093, "count": 13, "max_ms": 4.542, "p95_ms": 4.542}, "gstreamer_push": {"avg_ms": 1.283, "count": 13, "max_ms": 1.424, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.318, "count": 13, "max_ms": 83.959, "p95_ms": 83.959}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.085, "count": 13, "max_ms": 107.568, "p95_ms": 107.568}, "yolo11_640_fire_11class": {"avg_ms": 71.842, "count": 13, "max_ms": 89.791, "p95_ms": 89.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.192, "count": 13, "max_ms": 96.962, "p95_ms": 96.962}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.83, "count": 13, "max_ms": 120.407, "p95_ms": 120.407}, "read_interval": {"avg_ms": 2387.865, "count": 13, "max_ms": 16011.192, "p95_ms": 16011.192}, "triton_wait": {"avg_ms": 81.122, "count": 13, "max_ms": 107.582, "p95_ms": 107.582}, "visualize": {"avg_ms": 3.583, "count": 13, "max_ms": 3.68, "p95_ms": 3.68}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.287, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2458}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6059.191, "limit_mb": 8796093022207.996}, "detail": {"pid": 32434, "restart_count": 76, "start_count": 77}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:24:07+09:00", "last_error_summary": "[h264 @ 0x556d192ae680] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x556d18b66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d18b66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:24:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:24:09+09:00", "pid": 32434, "process": {"exists": true, "fd_count": 5, "hwm_mb": 62.375, "name": "ffmpeg", "pid": 32434, "rss_mb": 62.375, "state": "S (sleeping)", "threads": 19}, "restart_count": 76, "restart_count_per_hour": 19, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 34, "rtsp_tolerant_mode": false, "running": true, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.73, "sample_count": 2443, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295005, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:24:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.171, "last_frame_pushed_at": "2026-07-06T22:24:07+09:00", "last_frame_read_age_sec": 4.294, "last_frame_read_at": "2026-07-06T22:24:07+09:00", "mjpeg": {"age_sec": 4.153, "exists": true, "mtime": "2026-07-06T22:24:07.771585+09:00", "mtime_epoch": 1783344247.7715847, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98779}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:24:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1553.539, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.606, "pushed": 0.606, "read": 0.606}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.093, "count": 13, "max_ms": 4.542, "p95_ms": 4.542}, "gstreamer_push": {"avg_ms": 1.283, "count": 13, "max_ms": 1.424, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.318, "count": 13, "max_ms": 83.959, "p95_ms": 83.959}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.085, "count": 13, "max_ms": 107.568, "p95_ms": 107.568}, "yolo11_640_fire_11class": {"avg_ms": 71.842, "count": 13, "max_ms": 89.791, "p95_ms": 89.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.192, "count": 13, "max_ms": 96.962, "p95_ms": 96.962}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.83, "count": 13, "max_ms": 120.407, "p95_ms": 120.407}, "read_interval": {"avg_ms": 2387.865, "count": 13, "max_ms": 16011.192, "p95_ms": 16011.192}, "triton_wait": {"avg_ms": 81.122, "count": 13, "max_ms": 107.582, "p95_ms": 107.582}, "visualize": {"avg_ms": 3.583, "count": 13, "max_ms": 3.68, "p95_ms": 3.68}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.446, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2465}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6329.465, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:24:07+09:00", "last_error_summary": "[h264 @ 0x556d192ae680] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x556d18b66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d18b66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:24:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.506, "last_started_at": "2026-07-06T22:24:09+09:00", "pid": 32434, "process": {"exists": true, "fd_count": 5, "hwm_mb": 328.941, "name": "ffmpeg", "pid": 32434, "rss_mb": 328.941, "state": "S (sleeping)", "threads": 106}, "restart_count": 76, "restart_count_per_hour": 18, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": true, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.821, "sample_count": 2181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295849, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:25:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.057, "last_frame_pushed_at": "2026-07-06T22:25:46+09:00", "last_frame_read_age_sec": 30.154, "last_frame_read_at": "2026-07-06T22:25:46+09:00", "mjpeg": {"age_sec": 30.04, "exists": true, "mtime": "2026-07-06T22:25:46.389095+09:00", "mtime_epoch": 1783344346.3890948, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:26:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1554.055, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2465}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6315.062, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:24:07+09:00", "last_error_summary": "[h264 @ 0x556d192ae680] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x556d18b66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d18b66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:24:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.518, "last_started_at": "2026-07-06T22:24:09+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 76, "restart_count_per_hour": 18, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.821, "sample_count": 2181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295849, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:25:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.07, "last_frame_pushed_at": "2026-07-06T22:25:46+09:00", "last_frame_read_age_sec": 30.167, "last_frame_read_at": "2026-07-06T22:25:46+09:00", "mjpeg": {"age_sec": 30.052, "exists": true, "mtime": "2026-07-06T22:25:46.389095+09:00", "mtime_epoch": 1783344346.3890948, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:26:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1554.055, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2465}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6028.27, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ddcf3b59c0] error while decoding MB 39 21, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:26:16+09:00", "last_error_summary": "[h264 @ 0x55ddcf3b59c0] error while decoding MB 39 21, bytestream -7", "last_exit_at": "2026-07-06T22:24:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.574, "last_started_at": "2026-07-06T22:24:09+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 76, "restart_count_per_hour": 18, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.821, "sample_count": 2180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295849, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:25:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.126, "last_frame_pushed_at": "2026-07-06T22:25:46+09:00", "last_frame_read_age_sec": 30.223, "last_frame_read_at": "2026-07-06T22:25:46+09:00", "mjpeg": {"age_sec": 30.108, "exists": true, "mtime": "2026-07-06T22:25:46.389095+09:00", "mtime_epoch": 1783344346.3890948, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:26:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1554.055, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2465}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6048.379, "limit_mb": 8796093022207.996}, "detail": {"pid": 32586, "restart_count": 77, "start_count": 78}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:26:16+09:00", "last_error_summary": "[h264 @ 0x55ddcf3b59c0] error while decoding MB 39 21, bytestream -7", "last_exit_at": "2026-07-06T22:24:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:26:18+09:00", "pid": 32586, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.266, "name": "ffmpeg", "pid": 32586, "rss_mb": 58.266, "state": "S (sleeping)", "threads": 1}, "restart_count": 77, "restart_count_per_hour": 19, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 32, "rtsp_tolerant_mode": false, "running": true, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.821, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295849, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:25:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.32, "last_frame_pushed_at": "2026-07-06T22:25:46+09:00", "last_frame_read_age_sec": 34.417, "last_frame_read_at": "2026-07-06T22:25:46+09:00", "mjpeg": {"age_sec": 34.303, "exists": true, "mtime": "2026-07-06T22:25:46.389095+09:00", "mtime_epoch": 1783344346.3890948, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:26:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1554.055, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6058.406, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5646caebea40] error while decoding MB 67 32, bytestream -5 [in#0/rtsp @ 0x5646ca5e7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646ca5e7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:30:31+09:00", "last_error_summary": "[h264 @ 0x5646caebea40] error while decoding MB 67 32, bytestream -5 [in#0/rtsp @ 0x5646ca5e7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646ca5e7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:24:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 252.49, "last_started_at": "2026-07-06T22:26:18+09:00", "pid": 32586, "process": {"exists": false, "pid": 32586}, "restart_count": 77, "restart_count_per_hour": 17, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.569, "sample_count": 2189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 297902, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:30:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-06T22:30:31+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T22:30:31+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-06T22:30:31.125551+09:00", "mtime_epoch": 1783344631.1255512, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105458}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:30:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1554.828, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.771, "pushed": 2.771, "read": 2.771}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.18, "count": 83, "max_ms": 6.981, "p95_ms": 5.666}, "gstreamer_push": {"avg_ms": 1.25, "count": 83, "max_ms": 1.648, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.646, "count": 83, "max_ms": 89.924, "p95_ms": 79.735}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.539, "count": 83, "max_ms": 103.157, "p95_ms": 94.759}, "yolo11_640_fire_11class": {"avg_ms": 75.372, "count": 83, "max_ms": 89.944, "p95_ms": 85.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.037, "count": 83, "max_ms": 102.151, "p95_ms": 94.89}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.78, "count": 83, "max_ms": 115.693, "p95_ms": 111.213}, "read_interval": {"avg_ms": 361.122, "count": 83, "max_ms": 11960.612, "p95_ms": 127.87}, "triton_wait": {"avg_ms": 85.041, "count": 83, "max_ms": 103.17, "p95_ms": 97.787}, "visualize": {"avg_ms": 3.561, "count": 83, "max_ms": 5.972, "p95_ms": 3.973}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6057.879, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:30:31+09:00", "last_error_summary": "[h264 @ 0x5646caebea40] error while decoding MB 67 32, bytestream -5 [in#0/rtsp @ 0x5646ca5e7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646ca5e7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:30:31+09:00", "last_error_summary": "[h264 @ 0x5646caebea40] error while decoding MB 67 32, bytestream -5 [in#0/rtsp @ 0x5646ca5e7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646ca5e7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:30:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 252.537, "last_started_at": "2026-07-06T22:26:18+09:00", "pid": 32586, "process": {"exists": false, "pid": 32586}, "restart_count": 77, "restart_count_per_hour": 17, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.569, "sample_count": 2190, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 297903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:30:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:30:31+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T22:30:31+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T22:30:31.125551+09:00", "mtime_epoch": 1783344631.1255512, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105458}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:30:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1554.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.8, "pushed": 2.8, "read": 2.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.179, "count": 83, "max_ms": 6.981, "p95_ms": 5.666}, "gstreamer_push": {"avg_ms": 1.251, "count": 84, "max_ms": 1.648, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.016, "count": 83, "max_ms": 89.924, "p95_ms": 79.735}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.746, "count": 83, "max_ms": 103.157, "p95_ms": 94.759}, "yolo11_640_fire_11class": {"avg_ms": 75.455, "count": 83, "max_ms": 89.944, "p95_ms": 85.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.333, "count": 83, "max_ms": 102.151, "p95_ms": 94.89}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.824, "count": 84, "max_ms": 115.693, "p95_ms": 111.213}, "read_interval": {"avg_ms": 361.122, "count": 83, "max_ms": 11960.612, "p95_ms": 127.87}, "triton_wait": {"avg_ms": 85.234, "count": 83, "max_ms": 103.17, "p95_ms": 97.787}, "visualize": {"avg_ms": 3.563, "count": 83, "max_ms": 5.972, "p95_ms": 3.973}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6057.82, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:30:31+09:00", "last_error_summary": "[h264 @ 0x5646caebea40] error while decoding MB 67 32, bytestream -5 [in#0/rtsp @ 0x5646ca5e7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646ca5e7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:30:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 252.544, "last_started_at": "2026-07-06T22:26:18+09:00", "pid": 32586, "process": {"exists": false, "pid": 32586}, "restart_count": 77, "restart_count_per_hour": 17, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.569, "sample_count": 2190, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 297903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:30:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T22:30:31+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T22:30:31+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T22:30:31.125551+09:00", "mtime_epoch": 1783344631.1255512, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105458}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:30:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1554.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.768, "pushed": 2.768, "read": 2.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.179, "count": 83, "max_ms": 6.981, "p95_ms": 5.666}, "gstreamer_push": {"avg_ms": 1.25, "count": 83, "max_ms": 1.648, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.016, "count": 83, "max_ms": 89.924, "p95_ms": 79.735}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.746, "count": 83, "max_ms": 103.157, "p95_ms": 94.759}, "yolo11_640_fire_11class": {"avg_ms": 75.455, "count": 83, "max_ms": 89.944, "p95_ms": 85.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.333, "count": 83, "max_ms": 102.151, "p95_ms": 94.89}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.973, "count": 83, "max_ms": 115.693, "p95_ms": 111.213}, "read_interval": {"avg_ms": 361.122, "count": 83, "max_ms": 11960.612, "p95_ms": 127.87}, "triton_wait": {"avg_ms": 85.234, "count": 83, "max_ms": 103.17, "p95_ms": 97.787}, "visualize": {"avg_ms": 3.563, "count": 83, "max_ms": 5.972, "p95_ms": 3.973}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6227.113, "limit_mb": 8796093022207.996}, "detail": {"pid": 32738, "restart_count": 78, "start_count": 79}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:30:31+09:00", "last_error_summary": "[h264 @ 0x5646caebea40] error while decoding MB 67 32, bytestream -5 [in#0/rtsp @ 0x5646ca5e7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5646ca5e7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:30:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:30:33+09:00", "pid": 32738, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.469, "name": "ffmpeg", "pid": 32738, "rss_mb": 197.469, "state": "S (sleeping)", "threads": 106}, "restart_count": 78, "restart_count_per_hour": 18, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": true, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.458, "sample_count": 2153, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 297903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:30:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.193, "last_frame_pushed_at": "2026-07-06T22:30:31+09:00", "last_frame_read_age_sec": 4.296, "last_frame_read_at": "2026-07-06T22:30:31+09:00", "mjpeg": {"age_sec": 4.173, "exists": true, "mtime": "2026-07-06T22:30:31.241553+09:00", "mtime_epoch": 1783344631.241553, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:30:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1554.828, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 46, "pushed_frames": 46, "read_frames": 45, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.536, "pushed": 1.536, "read": 1.502}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.371, "count": 46, "max_ms": 6.981, "p95_ms": 6.715}, "gstreamer_push": {"avg_ms": 1.272, "count": 46, "max_ms": 1.648, "p95_ms": 1.429}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.199, "count": 46, "max_ms": 84.871, "p95_ms": 78.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.339, "count": 46, "max_ms": 103.157, "p95_ms": 94.759}, "yolo11_640_fire_11class": {"avg_ms": 74.631, "count": 46, "max_ms": 86.898, "p95_ms": 84.051}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.891, "count": 46, "max_ms": 95.766, "p95_ms": 90.402}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.323, "count": 46, "max_ms": 115.693, "p95_ms": 110.304}, "read_interval": {"avg_ms": 572.242, "count": 45, "max_ms": 11960.612, "p95_ms": 131.609}, "triton_wait": {"avg_ms": 84.128, "count": 46, "max_ms": 103.17, "p95_ms": 95.779}, "visualize": {"avg_ms": 3.709, "count": 46, "max_ms": 5.972, "p95_ms": 4.248}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2306}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6100.0, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559d6323d740] error while decoding MB 0 12, bytestream -6 [h264 @ 0x559d63136380] left block unavailable for requested intra mode [h264 @ 0x559d63136380] error while decoding MB 0 10, bytestream 71795 [h264 @ 0x559d63b82ac0] left block unavailable for requested intra mode [h264 @ 0x559d63b82ac0] error while decoding MB 0 27, bytestream 3435 [h264 @ 0x559d63ebe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d63ebe4c0] error while decoding MB 0 11, bytestream 671 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 121, "h264_decode_error_count": 402, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:31:28+09:00", "last_error_summary": "[h264 @ 0x559d6323d740] error while decoding MB 0 12, bytestream -6 [h264 @ 0x559d63136380] left block unavailable for requested intra mode [h264 @ 0x559d63136380] error while decoding MB 0 10, bytestream 71795 [h264 @ 0x559d63b82ac0] left block unavailable for requested intra mode [h264 @ 0x559d63b82ac0] error while decoding MB 0 27, bytestream 3435 [h264 @ 0x559d63ebe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d63ebe4c0] error while decoding MB 0 11, bytestream 671 ", "last_exit_at": "2026-07-06T22:23:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 495.22, "last_started_at": "2026-07-06T22:23:13+09:00", "pid": 32327, "process": {"exists": false, "pid": 32327}, "restart_count": 130, "restart_count_per_hour": 5, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 131}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.849, "sample_count": 2576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 276798, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:31:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-06T22:31:28+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T22:31:28+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-06T22:31:28.394460+09:00", "mtime_epoch": 1783344688.3944604, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:31:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1671.117, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.269, "pushed": 3.269, "read": 3.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.855, "count": 98, "max_ms": 5.806, "p95_ms": 5.768}, "gstreamer_push": {"avg_ms": 1.453, "count": 98, "max_ms": 2.431, "p95_ms": 2.251}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.272, "count": 98, "max_ms": 89.033, "p95_ms": 82.344}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.7, "count": 98, "max_ms": 118.446, "p95_ms": 103.631}, "yolo11_640_fire_11class": {"avg_ms": 75.191, "count": 98, "max_ms": 89.045, "p95_ms": 84.409}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.766, "count": 98, "max_ms": 97.989, "p95_ms": 95.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.083, "count": 98, "max_ms": 130.665, "p95_ms": 117.452}, "read_interval": {"avg_ms": 306.519, "count": 98, "max_ms": 10119.617, "p95_ms": 138.064}, "triton_wait": {"avg_ms": 84.87, "count": 98, "max_ms": 118.458, "p95_ms": 103.642}, "visualize": {"avg_ms": 4.013, "count": 98, "max_ms": 6.387, "p95_ms": 6.3}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2306}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6099.668, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:31:28+09:00", "last_error_summary": "[h264 @ 0x559d6323d740] error while decoding MB 0 12, bytestream -6 [h264 @ 0x559d63136380] left block unavailable for requested intra mode [h264 @ 0x559d63136380] error while decoding MB 0 10, bytestream 71795 [h264 @ 0x559d63b82ac0] left block unavailable for requested intra mode [h264 @ 0x559d63b82ac0] error while decoding MB 0 27, bytestream 3435 [h264 @ 0x559d63ebe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d63ebe4c0] error while decoding MB 0 11, bytestream 671 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 121, "h264_decode_error_count": 402, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:31:28+09:00", "last_error_summary": "[h264 @ 0x559d6323d740] error while decoding MB 0 12, bytestream -6 [h264 @ 0x559d63136380] left block unavailable for requested intra mode [h264 @ 0x559d63136380] error while decoding MB 0 10, bytestream 71795 [h264 @ 0x559d63b82ac0] left block unavailable for requested intra mode [h264 @ 0x559d63b82ac0] error while decoding MB 0 27, bytestream 3435 [h264 @ 0x559d63ebe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d63ebe4c0] error while decoding MB 0 11, bytestream 671 ", "last_exit_at": "2026-07-06T22:31:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 495.271, "last_started_at": "2026-07-06T22:23:13+09:00", "pid": 32327, "process": {"exists": false, "pid": 32327}, "restart_count": 130, "restart_count_per_hour": 5, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 131}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.849, "sample_count": 2577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 276799, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:31:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:31:28+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-06T22:31:28+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-06T22:31:28.394460+09:00", "mtime_epoch": 1783344688.3944604, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:31:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1671.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.277, "pushed": 3.277, "read": 3.244}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.826, "count": 98, "max_ms": 5.806, "p95_ms": 5.768}, "gstreamer_push": {"avg_ms": 1.442, "count": 98, "max_ms": 2.431, "p95_ms": 2.244}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.189, "count": 98, "max_ms": 89.033, "p95_ms": 82.344}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.657, "count": 98, "max_ms": 118.446, "p95_ms": 103.631}, "yolo11_640_fire_11class": {"avg_ms": 75.17, "count": 98, "max_ms": 89.045, "p95_ms": 84.409}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.689, "count": 98, "max_ms": 97.989, "p95_ms": 95.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.915, "count": 98, "max_ms": 130.665, "p95_ms": 117.452}, "read_interval": {"avg_ms": 308.308, "count": 97, "max_ms": 10119.617, "p95_ms": 138.064}, "triton_wait": {"avg_ms": 84.793, "count": 98, "max_ms": 118.458, "p95_ms": 103.642}, "visualize": {"avg_ms": 3.98, "count": 98, "max_ms": 6.387, "p95_ms": 6.298}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2306}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6099.641, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 121, "h264_decode_error_count": 402, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:31:28+09:00", "last_error_summary": "[h264 @ 0x559d6323d740] error while decoding MB 0 12, bytestream -6 [h264 @ 0x559d63136380] left block unavailable for requested intra mode [h264 @ 0x559d63136380] error while decoding MB 0 10, bytestream 71795 [h264 @ 0x559d63b82ac0] left block unavailable for requested intra mode [h264 @ 0x559d63b82ac0] error while decoding MB 0 27, bytestream 3435 [h264 @ 0x559d63ebe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d63ebe4c0] error while decoding MB 0 11, bytestream 671 ", "last_exit_at": "2026-07-06T22:31:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 495.28, "last_started_at": "2026-07-06T22:23:13+09:00", "pid": 32327, "process": {"exists": false, "pid": 32327}, "restart_count": 130, "restart_count_per_hour": 5, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 131}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.849, "sample_count": 2577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 276799, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:31:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T22:31:28+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T22:31:28+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-06T22:31:28.394460+09:00", "mtime_epoch": 1783344688.3944604, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:31:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1671.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.276, "pushed": 3.276, "read": 3.243}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.826, "count": 98, "max_ms": 5.806, "p95_ms": 5.768}, "gstreamer_push": {"avg_ms": 1.442, "count": 98, "max_ms": 2.431, "p95_ms": 2.244}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.189, "count": 98, "max_ms": 89.033, "p95_ms": 82.344}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.657, "count": 98, "max_ms": 118.446, "p95_ms": 103.631}, "yolo11_640_fire_11class": {"avg_ms": 75.17, "count": 98, "max_ms": 89.045, "p95_ms": 84.409}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.689, "count": 98, "max_ms": 97.989, "p95_ms": 95.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.915, "count": 98, "max_ms": 130.665, "p95_ms": 117.452}, "read_interval": {"avg_ms": 308.308, "count": 97, "max_ms": 10119.617, "p95_ms": 138.064}, "triton_wait": {"avg_ms": 84.793, "count": 98, "max_ms": 118.458, "p95_ms": 103.642}, "visualize": {"avg_ms": 3.98, "count": 98, "max_ms": 6.387, "p95_ms": 6.298}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2306}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6209.609, "limit_mb": 8796093022207.996}, "detail": {"pid": 32890, "restart_count": 131, "start_count": 132}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 121, "h264_decode_error_count": 402, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:31:28+09:00", "last_error_summary": "[h264 @ 0x559d6323d740] error while decoding MB 0 12, bytestream -6 [h264 @ 0x559d63136380] left block unavailable for requested intra mode [h264 @ 0x559d63136380] error while decoding MB 0 10, bytestream 71795 [h264 @ 0x559d63b82ac0] left block unavailable for requested intra mode [h264 @ 0x559d63b82ac0] error while decoding MB 0 27, bytestream 3435 [h264 @ 0x559d63ebe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d63ebe4c0] error while decoding MB 0 11, bytestream 671 ", "last_exit_at": "2026-07-06T22:31:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:31:30+09:00", "pid": 32890, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.352, "name": "ffmpeg", "pid": 32890, "rss_mb": 141.352, "state": "S (sleeping)", "threads": 106}, "restart_count": 131, "restart_count_per_hour": 6, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 132}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.508, "sample_count": 2538, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 276799, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:31:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.367, "last_frame_pushed_at": "2026-07-06T22:31:28+09:00", "last_frame_read_age_sec": 4.467, "last_frame_read_at": "2026-07-06T22:31:28+09:00", "mjpeg": {"age_sec": 4.349, "exists": true, "mtime": "2026-07-06T22:31:28.506462+09:00", "mtime_epoch": 1783344688.5064623, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120640}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:31:32+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1671.117, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.001, "pushed": 2.001, "read": 1.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.706, "count": 60, "max_ms": 5.806, "p95_ms": 5.771}, "gstreamer_push": {"avg_ms": 1.406, "count": 60, "max_ms": 2.355, "p95_ms": 2.189}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.845, "count": 60, "max_ms": 89.033, "p95_ms": 82.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.69, "count": 60, "max_ms": 118.446, "p95_ms": 107.287}, "yolo11_640_fire_11class": {"avg_ms": 74.655, "count": 60, "max_ms": 89.045, "p95_ms": 84.368}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.917, "count": 60, "max_ms": 97.989, "p95_ms": 95.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.45, "count": 60, "max_ms": 130.665, "p95_ms": 119.443}, "read_interval": {"avg_ms": 434.095, "count": 59, "max_ms": 10119.617, "p95_ms": 153.986}, "triton_wait": {"avg_ms": 84.67, "count": 60, "max_ms": 118.458, "p95_ms": 107.299}, "visualize": {"avg_ms": 3.857, "count": 60, "max_ms": 6.367, "p95_ms": 6.277}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6035.953, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56372e5dd400] error while decoding MB 90 47, bytestream -5 [in#0/rtsp @ 0x56372d9c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56372d9c8e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:33:45+09:00", "last_error_summary": "[h264 @ 0x56372e5dd400] error while decoding MB 90 47, bytestream -5 [in#0/rtsp @ 0x56372d9c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56372d9c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:30:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.738, "last_started_at": "2026-07-06T22:30:33+09:00", "pid": 32738, "process": {"exists": false, "pid": 32738}, "restart_count": 78, "restart_count_per_hour": 17, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.458, "sample_count": 2240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299371, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:33:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-06T22:33:45+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T22:33:45+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-06T22:33:45.096644+09:00", "mtime_epoch": 1783344825.096644, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104783}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:33:45+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1555.602, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.743, "pushed": 0.743, "read": 0.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.212, "count": 13, "max_ms": 5.467, "p95_ms": 5.467}, "gstreamer_push": {"avg_ms": 1.267, "count": 13, "max_ms": 1.503, "p95_ms": 1.503}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.712, "count": 13, "max_ms": 80.134, "p95_ms": 80.134}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.395, "count": 13, "max_ms": 102.356, "p95_ms": 102.356}, "yolo11_640_fire_11class": {"avg_ms": 67.728, "count": 13, "max_ms": 86.798, "p95_ms": 86.798}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.359, "count": 13, "max_ms": 91.665, "p95_ms": 91.665}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.261, "count": 13, "max_ms": 115.458, "p95_ms": 115.458}, "read_interval": {"avg_ms": 2422.417, "count": 14, "max_ms": 16487.921, "p95_ms": 16487.921}, "triton_wait": {"avg_ms": 76.605, "count": 13, "max_ms": 102.369, "p95_ms": 102.369}, "visualize": {"avg_ms": 3.472, "count": 13, "max_ms": 3.921, "p95_ms": 3.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.489, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6035.508, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:33:45+09:00", "last_error_summary": "[h264 @ 0x56372e5dd400] error while decoding MB 90 47, bytestream -5 [in#0/rtsp @ 0x56372d9c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56372d9c8e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:33:45+09:00", "last_error_summary": "[h264 @ 0x56372e5dd400] error while decoding MB 90 47, bytestream -5 [in#0/rtsp @ 0x56372d9c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56372d9c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:33:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.775, "last_started_at": "2026-07-06T22:30:33+09:00", "pid": 32738, "process": {"exists": false, "pid": 32738}, "restart_count": 78, "restart_count_per_hour": 17, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.458, "sample_count": 2240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:33:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:33:45+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-06T22:33:45+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-06T22:33:45.096644+09:00", "mtime_epoch": 1783344825.096644, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104783}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:33:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1555.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.799, "pushed": 0.799, "read": 0.799}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.293, "count": 14, "max_ms": 5.467, "p95_ms": 5.467}, "gstreamer_push": {"avg_ms": 1.255, "count": 14, "max_ms": 1.503, "p95_ms": 1.503}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.997, "count": 14, "max_ms": 80.134, "p95_ms": 80.134}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.93, "count": 14, "max_ms": 102.356, "p95_ms": 102.356}, "yolo11_640_fire_11class": {"avg_ms": 68.504, "count": 14, "max_ms": 86.798, "p95_ms": 86.798}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.04, "count": 14, "max_ms": 91.665, "p95_ms": 91.665}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.464, "count": 14, "max_ms": 115.458, "p95_ms": 115.458}, "read_interval": {"avg_ms": 2422.417, "count": 14, "max_ms": 16487.921, "p95_ms": 16487.921}, "triton_wait": {"avg_ms": 76.748, "count": 14, "max_ms": 102.369, "p95_ms": 102.369}, "visualize": {"avg_ms": 3.476, "count": 14, "max_ms": 3.921, "p95_ms": 3.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.526, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6035.668, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:33:45+09:00", "last_error_summary": "[h264 @ 0x56372e5dd400] error while decoding MB 90 47, bytestream -5 [in#0/rtsp @ 0x56372d9c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56372d9c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:33:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.783, "last_started_at": "2026-07-06T22:30:33+09:00", "pid": 32738, "process": {"exists": false, "pid": 32738}, "restart_count": 78, "restart_count_per_hour": 17, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.458, "sample_count": 2240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:33:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T22:33:45+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T22:33:45+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T22:33:45.096644+09:00", "mtime_epoch": 1783344825.096644, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104783}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:33:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1555.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.798, "pushed": 0.798, "read": 0.798}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.293, "count": 14, "max_ms": 5.467, "p95_ms": 5.467}, "gstreamer_push": {"avg_ms": 1.255, "count": 14, "max_ms": 1.503, "p95_ms": 1.503}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.997, "count": 14, "max_ms": 80.134, "p95_ms": 80.134}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.93, "count": 14, "max_ms": 102.356, "p95_ms": 102.356}, "yolo11_640_fire_11class": {"avg_ms": 68.504, "count": 14, "max_ms": 86.798, "p95_ms": 86.798}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.04, "count": 14, "max_ms": 91.665, "p95_ms": 91.665}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.464, "count": 14, "max_ms": 115.458, "p95_ms": 115.458}, "read_interval": {"avg_ms": 2422.417, "count": 14, "max_ms": 16487.921, "p95_ms": 16487.921}, "triton_wait": {"avg_ms": 76.748, "count": 14, "max_ms": 102.369, "p95_ms": 102.369}, "visualize": {"avg_ms": 3.476, "count": 14, "max_ms": 3.921, "p95_ms": 3.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.534, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6150.68, "limit_mb": 8796093022207.996}, "detail": {"pid": 32997, "restart_count": 79, "start_count": 80}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:33:45+09:00", "last_error_summary": "[h264 @ 0x56372e5dd400] error while decoding MB 90 47, bytestream -5 [in#0/rtsp @ 0x56372d9c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56372d9c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:33:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:33:48+09:00", "pid": 32997, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.938, "name": "ffmpeg", "pid": 32997, "rss_mb": 147.938, "state": "R (running)", "threads": 49}, "restart_count": 79, "restart_count_per_hour": 18, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 30, "rtsp_tolerant_mode": false, "running": true, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.57, "sample_count": 2196, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:33:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "rtsp_timeout_count_per_hour>=30", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.876, "last_frame_pushed_at": "2026-07-06T22:33:45+09:00", "last_frame_read_age_sec": 4.97, "last_frame_read_at": "2026-07-06T22:33:45+09:00", "mjpeg": {"age_sec": 4.862, "exists": true, "mtime": "2026-07-06T22:33:45.196645+09:00", "mtime_epoch": 1783344825.1966455, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97954}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:33:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1555.602, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.625, "pushed": 0.625, "read": 0.625}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.293, "count": 14, "max_ms": 5.467, "p95_ms": 5.467}, "gstreamer_push": {"avg_ms": 1.255, "count": 14, "max_ms": 1.503, "p95_ms": 1.503}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.997, "count": 14, "max_ms": 80.134, "p95_ms": 80.134}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.93, "count": 14, "max_ms": 102.356, "p95_ms": 102.356}, "yolo11_640_fire_11class": {"avg_ms": 68.504, "count": 14, "max_ms": 86.798, "p95_ms": 86.798}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.04, "count": 14, "max_ms": 91.665, "p95_ms": 91.665}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.464, "count": 14, "max_ms": 115.458, "p95_ms": 115.458}, "read_interval": {"avg_ms": 2422.417, "count": 14, "max_ms": 16487.921, "p95_ms": 16487.921}, "triton_wait": {"avg_ms": 76.748, "count": 14, "max_ms": 102.369, "p95_ms": 102.369}, "visualize": {"avg_ms": 3.476, "count": 14, "max_ms": 3.921, "p95_ms": 3.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.396, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2342}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6097.816, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562b6264dcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b6264dcc0] error while decoding MB 0 4, bytestream 5345 [h264 @ 0x562b63036f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b63036f40] error while decoding MB 0 7, bytestream 5879 [h264 @ 0x562b625720c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b625720c0] error while decoding MB 0 5, bytestream 5336 [in#0/rtsp @ 0x562b624eee00] Error during demuxing: Connect", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 122, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:39:42+09:00", "last_error_summary": "[h264 @ 0x562b6264dcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b6264dcc0] error while decoding MB 0 4, bytestream 5345 [h264 @ 0x562b63036f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b63036f40] error while decoding MB 0 7, bytestream 5879 [h264 @ 0x562b625720c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b625720c0] error while decoding MB 0 5, bytestream 5336 [in#0/rtsp @ 0x562b624eee00] Error during demuxing: Connect", "last_exit_at": "2026-07-06T22:31:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 491.207, "last_started_at": "2026-07-06T22:31:30+09:00", "pid": 32890, "process": {"exists": false, "pid": 32890}, "restart_count": 131, "restart_count_per_hour": 5, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 132}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.721, "sample_count": 2522, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281112, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:39:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-06T22:39:42+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-06T22:39:42+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-06T22:39:42.022405+09:00", "mtime_epoch": 1783345182.0224051, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:39:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1673.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.275, "pushed": 3.275, "read": 3.275}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.59, "count": 98, "max_ms": 5.216, "p95_ms": 4.77}, "gstreamer_push": {"avg_ms": 1.35, "count": 98, "max_ms": 1.866, "p95_ms": 1.492}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.482, "count": 98, "max_ms": 87.47, "p95_ms": 81.834}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.537, "count": 98, "max_ms": 114.695, "p95_ms": 96.456}, "yolo11_640_fire_11class": {"avg_ms": 73.383, "count": 98, "max_ms": 88.672, "p95_ms": 86.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.491, "count": 98, "max_ms": 100.339, "p95_ms": 97.12}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.934, "count": 98, "max_ms": 127.312, "p95_ms": 112.721}, "read_interval": {"avg_ms": 305.875, "count": 98, "max_ms": 10124.389, "p95_ms": 135.778}, "triton_wait": {"avg_ms": 85.584, "count": 98, "max_ms": 114.707, "p95_ms": 100.351}, "visualize": {"avg_ms": 3.66, "count": 98, "max_ms": 4.672, "p95_ms": 4.152}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2342}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6097.199, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:39:42+09:00", "last_error_summary": "[h264 @ 0x562b6264dcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b6264dcc0] error while decoding MB 0 4, bytestream 5345 [h264 @ 0x562b63036f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b63036f40] error while decoding MB 0 7, bytestream 5879 [h264 @ 0x562b625720c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b625720c0] error while decoding MB 0 5, bytestream 5336 [in#0/rtsp @ 0x562b624eee00] Error during demuxing: Connect", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 122, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:39:42+09:00", "last_error_summary": "[h264 @ 0x562b6264dcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b6264dcc0] error while decoding MB 0 4, bytestream 5345 [h264 @ 0x562b63036f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b63036f40] error while decoding MB 0 7, bytestream 5879 [h264 @ 0x562b625720c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b625720c0] error while decoding MB 0 5, bytestream 5336 [in#0/rtsp @ 0x562b624eee00] Error during demuxing: Connect", "last_exit_at": "2026-07-06T22:39:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 491.249, "last_started_at": "2026-07-06T22:31:30+09:00", "pid": 32890, "process": {"exists": false, "pid": 32890}, "restart_count": 131, "restart_count_per_hour": 5, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 132}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.721, "sample_count": 2523, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281113, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:39:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:39:42+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-06T22:39:42+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-06T22:39:42.022405+09:00", "mtime_epoch": 1783345182.0224051, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:39:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1673.18, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.304, "pushed": 3.304, "read": 3.271}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.588, "count": 99, "max_ms": 5.216, "p95_ms": 4.77}, "gstreamer_push": {"avg_ms": 1.35, "count": 99, "max_ms": 1.866, "p95_ms": 1.492}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.18, "count": 99, "max_ms": 87.47, "p95_ms": 81.834}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.457, "count": 99, "max_ms": 114.695, "p95_ms": 96.456}, "yolo11_640_fire_11class": {"avg_ms": 73.337, "count": 99, "max_ms": 88.672, "p95_ms": 86.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.339, "count": 99, "max_ms": 100.339, "p95_ms": 97.12}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.78, "count": 99, "max_ms": 127.312, "p95_ms": 112.721}, "read_interval": {"avg_ms": 305.875, "count": 98, "max_ms": 10124.389, "p95_ms": 135.778}, "triton_wait": {"avg_ms": 85.433, "count": 99, "max_ms": 114.707, "p95_ms": 100.351}, "visualize": {"avg_ms": 3.66, "count": 99, "max_ms": 4.672, "p95_ms": 4.152}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2342}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6097.328, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 122, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:39:42+09:00", "last_error_summary": "[h264 @ 0x562b6264dcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b6264dcc0] error while decoding MB 0 4, bytestream 5345 [h264 @ 0x562b63036f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b63036f40] error while decoding MB 0 7, bytestream 5879 [h264 @ 0x562b625720c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b625720c0] error while decoding MB 0 5, bytestream 5336 [in#0/rtsp @ 0x562b624eee00] Error during demuxing: Connect", "last_exit_at": "2026-07-06T22:39:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 491.257, "last_started_at": "2026-07-06T22:31:30+09:00", "pid": 32890, "process": {"exists": false, "pid": 32890}, "restart_count": 131, "restart_count_per_hour": 5, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 132}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.721, "sample_count": 2523, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281113, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:39:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T22:39:42+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-06T22:39:42+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-06T22:39:42.022405+09:00", "mtime_epoch": 1783345182.0224051, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:39:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1673.18, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.303, "pushed": 3.303, "read": 3.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.588, "count": 99, "max_ms": 5.216, "p95_ms": 4.77}, "gstreamer_push": {"avg_ms": 1.35, "count": 99, "max_ms": 1.866, "p95_ms": 1.492}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.18, "count": 99, "max_ms": 87.47, "p95_ms": 81.834}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.457, "count": 99, "max_ms": 114.695, "p95_ms": 96.456}, "yolo11_640_fire_11class": {"avg_ms": 73.337, "count": 99, "max_ms": 88.672, "p95_ms": 86.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.339, "count": 99, "max_ms": 100.339, "p95_ms": 97.12}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.78, "count": 99, "max_ms": 127.312, "p95_ms": 112.721}, "read_interval": {"avg_ms": 305.875, "count": 98, "max_ms": 10124.389, "p95_ms": 135.778}, "triton_wait": {"avg_ms": 85.433, "count": 99, "max_ms": 114.707, "p95_ms": 100.351}, "visualize": {"avg_ms": 3.66, "count": 99, "max_ms": 4.672, "p95_ms": 4.152}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 2342}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6201.285, "limit_mb": 8796093022207.996}, "detail": {"pid": 33149, "restart_count": 132, "start_count": 133}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 122, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:39:42+09:00", "last_error_summary": "[h264 @ 0x562b6264dcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b6264dcc0] error while decoding MB 0 4, bytestream 5345 [h264 @ 0x562b63036f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b63036f40] error while decoding MB 0 7, bytestream 5879 [h264 @ 0x562b625720c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x562b625720c0] error while decoding MB 0 5, bytestream 5336 [in#0/rtsp @ 0x562b624eee00] Error during demuxing: Connect", "last_exit_at": "2026-07-06T22:39:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:39:44+09:00", "pid": 33149, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.902, "name": "ffmpeg", "pid": 33149, "rss_mb": 138.902, "state": "S (sleeping)", "threads": 106}, "restart_count": 132, "restart_count_per_hour": 6, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 133}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.761, "sample_count": 2479, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281113, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:39:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.712, "last_frame_pushed_at": "2026-07-06T22:39:42+09:00", "last_frame_read_age_sec": 4.797, "last_frame_read_at": "2026-07-06T22:39:42+09:00", "mjpeg": {"age_sec": 4.687, "exists": true, "mtime": "2026-07-06T22:39:42.122407+09:00", "mtime_epoch": 1783345182.1224067, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121858}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:39:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1756.938, "name": "celery", "pid": 237, "rss_mb": 1673.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.902, "pushed": 1.902, "read": 1.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.655, "count": 57, "max_ms": 5.216, "p95_ms": 4.863}, "gstreamer_push": {"avg_ms": 1.357, "count": 57, "max_ms": 1.504, "p95_ms": 1.495}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.384, "count": 57, "max_ms": 85.56, "p95_ms": 79.827}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.218, "count": 57, "max_ms": 114.695, "p95_ms": 96.456}, "yolo11_640_fire_11class": {"avg_ms": 71.814, "count": 57, "max_ms": 88.672, "p95_ms": 86.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.855, "count": 57, "max_ms": 100.339, "p95_ms": 97.523}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.724, "count": 57, "max_ms": 127.312, "p95_ms": 112.721}, "read_interval": {"avg_ms": 451.041, "count": 56, "max_ms": 10124.389, "p95_ms": 145.682}, "triton_wait": {"avg_ms": 84.257, "count": 57, "max_ms": 114.707, "p95_ms": 100.351}, "visualize": {"avg_ms": 3.688, "count": 57, "max_ms": 4.672, "p95_ms": 4.484}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2535}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6048.332, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55dcd751b8c0] error while decoding MB 102 38, bytestream -15 [in#0/rtsp @ 0x55dcd7395e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcd7395e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:42:58+09:00", "last_error_summary": "[h264 @ 0x55dcd751b8c0] error while decoding MB 102 38, bytestream -15 [in#0/rtsp @ 0x55dcd7395e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcd7395e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:33:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 550.66, "last_started_at": "2026-07-06T22:33:48+09:00", "pid": 32997, "process": {"exists": false, "pid": 32997}, "restart_count": 79, "restart_count_per_hour": 16, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.008, "sample_count": 2593, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 304200, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:42:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-06T22:42:58+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T22:42:58+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T22:42:58.658999+09:00", "mtime_epoch": 1783345378.6589987, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104141}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:42:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1557.922, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.399, "count": 99, "max_ms": 6.401, "p95_ms": 5.876}, "gstreamer_push": {"avg_ms": 1.301, "count": 99, "max_ms": 1.475, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.316, "count": 99, "max_ms": 87.342, "p95_ms": 83.298}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.599, "count": 99, "max_ms": 114.733, "p95_ms": 101.743}, "yolo11_640_fire_11class": {"avg_ms": 74.862, "count": 99, "max_ms": 87.944, "p95_ms": 84.878}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.111, "count": 99, "max_ms": 97.726, "p95_ms": 96.292}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.377, "count": 99, "max_ms": 127.757, "p95_ms": 115.938}, "read_interval": {"avg_ms": 303.289, "count": 99, "max_ms": 10168.919, "p95_ms": 134.474}, "triton_wait": {"avg_ms": 85.223, "count": 99, "max_ms": 114.746, "p95_ms": 101.76}, "visualize": {"avg_ms": 3.696, "count": 99, "max_ms": 4.612, "p95_ms": 4.452}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2535}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6047.281, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:42:58+09:00", "last_error_summary": "[h264 @ 0x55dcd751b8c0] error while decoding MB 102 38, bytestream -15 [in#0/rtsp @ 0x55dcd7395e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcd7395e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:42:58+09:00", "last_error_summary": "[h264 @ 0x55dcd751b8c0] error while decoding MB 102 38, bytestream -15 [in#0/rtsp @ 0x55dcd7395e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcd7395e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:42:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 550.714, "last_started_at": "2026-07-06T22:33:48+09:00", "pid": 32997, "process": {"exists": false, "pid": 32997}, "restart_count": 79, "restart_count_per_hour": 16, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.008, "sample_count": 2593, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 304201, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:42:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:42:58+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T22:42:58+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T22:42:58.658999+09:00", "mtime_epoch": 1783345378.6589987, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104141}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:42:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1557.922, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.307, "pushed": 3.307, "read": 3.273}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.399, "count": 99, "max_ms": 6.401, "p95_ms": 5.876}, "gstreamer_push": {"avg_ms": 1.301, "count": 99, "max_ms": 1.475, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.405, "count": 99, "max_ms": 87.342, "p95_ms": 83.352}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.693, "count": 99, "max_ms": 114.733, "p95_ms": 101.743}, "yolo11_640_fire_11class": {"avg_ms": 74.957, "count": 99, "max_ms": 87.944, "p95_ms": 85.717}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.184, "count": 99, "max_ms": 97.726, "p95_ms": 96.292}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.444, "count": 99, "max_ms": 127.757, "p95_ms": 115.938}, "read_interval": {"avg_ms": 305.173, "count": 98, "max_ms": 10168.919, "p95_ms": 134.474}, "triton_wait": {"avg_ms": 85.295, "count": 99, "max_ms": 114.746, "p95_ms": 101.76}, "visualize": {"avg_ms": 3.693, "count": 99, "max_ms": 4.612, "p95_ms": 4.452}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2535}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6047.25, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:42:58+09:00", "last_error_summary": "[h264 @ 0x55dcd751b8c0] error while decoding MB 102 38, bytestream -15 [in#0/rtsp @ 0x55dcd7395e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcd7395e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:42:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 550.722, "last_started_at": "2026-07-06T22:33:48+09:00", "pid": 32997, "process": {"exists": false, "pid": 32997}, "restart_count": 79, "restart_count_per_hour": 16, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.008, "sample_count": 2593, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 304201, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:42:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T22:42:58+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-06T22:42:58+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-06T22:42:58.658999+09:00", "mtime_epoch": 1783345378.6589987, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104141}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:42:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1557.922, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.306, "pushed": 3.306, "read": 3.272}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.399, "count": 99, "max_ms": 6.401, "p95_ms": 5.876}, "gstreamer_push": {"avg_ms": 1.301, "count": 99, "max_ms": 1.475, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.405, "count": 99, "max_ms": 87.342, "p95_ms": 83.352}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.693, "count": 99, "max_ms": 114.733, "p95_ms": 101.743}, "yolo11_640_fire_11class": {"avg_ms": 74.957, "count": 99, "max_ms": 87.944, "p95_ms": 85.717}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.184, "count": 99, "max_ms": 97.726, "p95_ms": 96.292}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.444, "count": 99, "max_ms": 127.757, "p95_ms": 115.938}, "read_interval": {"avg_ms": 305.173, "count": 98, "max_ms": 10168.919, "p95_ms": 134.474}, "triton_wait": {"avg_ms": 85.295, "count": 99, "max_ms": 114.746, "p95_ms": 101.76}, "visualize": {"avg_ms": 3.693, "count": 99, "max_ms": 4.612, "p95_ms": 4.452}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2535}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6218.305, "limit_mb": 8796093022207.996}, "detail": {"pid": 33316, "restart_count": 80, "start_count": 81}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:42:58+09:00", "last_error_summary": "[h264 @ 0x55dcd751b8c0] error while decoding MB 102 38, bytestream -15 [in#0/rtsp @ 0x55dcd7395e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dcd7395e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:43:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:43:01+09:00", "pid": 33316, "process": {"exists": true, "fd_count": 5, "hwm_mb": 198.523, "name": "ffmpeg", "pid": 33316, "rss_mb": 198.523, "state": "S (sleeping)", "threads": 106}, "restart_count": 80, "restart_count_per_hour": 17, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.35, "sample_count": 2551, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 304201, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:42:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.549, "last_frame_pushed_at": "2026-07-06T22:42:58+09:00", "last_frame_read_age_sec": 4.658, "last_frame_read_at": "2026-07-06T22:42:58+09:00", "mjpeg": {"age_sec": 4.532, "exists": true, "mtime": "2026-07-06T22:42:58.782999+09:00", "mtime_epoch": 1783345378.7829993, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104330}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:43:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1557.922, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.969, "pushed": 1.969, "read": 1.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.179, "count": 59, "max_ms": 5.059, "p95_ms": 4.981}, "gstreamer_push": {"avg_ms": 1.301, "count": 59, "max_ms": 1.475, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.902, "count": 59, "max_ms": 86.789, "p95_ms": 83.352}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.704, "count": 59, "max_ms": 114.733, "p95_ms": 106.551}, "yolo11_640_fire_11class": {"avg_ms": 75.367, "count": 59, "max_ms": 86.777, "p95_ms": 84.878}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.483, "count": 59, "max_ms": 97.559, "p95_ms": 96.079}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.744, "count": 59, "max_ms": 127.757, "p95_ms": 119.426}, "read_interval": {"avg_ms": 438.482, "count": 58, "max_ms": 10168.919, "p95_ms": 140.47}, "triton_wait": {"avg_ms": 85.894, "count": 59, "max_ms": 114.746, "p95_ms": 106.563}, "visualize": {"avg_ms": 3.637, "count": 59, "max_ms": 4.325, "p95_ms": 3.87}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6073.844, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c91b825700] error while decoding MB 31 21, bytestream -5 [in#0/rtsp @ 0x55c91b022e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c91b022e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:47:24+09:00", "last_error_summary": "[h264 @ 0x55c91b825700] error while decoding MB 31 21, bytestream -5 [in#0/rtsp @ 0x55c91b022e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c91b022e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:43:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.753, "last_started_at": "2026-07-06T22:43:01+09:00", "pid": 33316, "process": {"exists": false, "pid": 33316}, "restart_count": 80, "restart_count_per_hour": 15, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.159, "sample_count": 2133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 306192, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:47:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-06T22:47:23+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T22:47:24+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-06T22:47:24.004797+09:00", "mtime_epoch": 1783345644.0047965, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95504}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:47:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1558.695, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.745, "pushed": 0.745, "read": 0.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.155, "count": 13, "max_ms": 4.656, "p95_ms": 4.656}, "gstreamer_push": {"avg_ms": 1.285, "count": 13, "max_ms": 1.414, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.902, "count": 13, "max_ms": 85.108, "p95_ms": 85.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.871, "count": 13, "max_ms": 87.501, "p95_ms": 87.501}, "yolo11_640_fire_11class": {"avg_ms": 66.886, "count": 13, "max_ms": 84.573, "p95_ms": 84.573}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.294, "count": 13, "max_ms": 94.472, "p95_ms": 94.472}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.118, "count": 13, "max_ms": 107.385, "p95_ms": 107.385}, "read_interval": {"avg_ms": 2415.866, "count": 14, "max_ms": 16430.88, "p95_ms": 16430.88}, "triton_wait": {"avg_ms": 72.282, "count": 13, "max_ms": 94.484, "p95_ms": 94.484}, "visualize": {"avg_ms": 3.63, "count": 13, "max_ms": 3.852, "p95_ms": 3.852}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.453, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6073.109, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:47:24+09:00", "last_error_summary": "[h264 @ 0x55c91b825700] error while decoding MB 31 21, bytestream -5 [in#0/rtsp @ 0x55c91b022e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c91b022e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:47:24+09:00", "last_error_summary": "[h264 @ 0x55c91b825700] error while decoding MB 31 21, bytestream -5 [in#0/rtsp @ 0x55c91b022e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c91b022e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:47:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.801, "last_started_at": "2026-07-06T22:43:01+09:00", "pid": 33316, "process": {"exists": false, "pid": 33316}, "restart_count": 80, "restart_count_per_hour": 15, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.159, "sample_count": 2133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 306193, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:47:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:47:24+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T22:47:24+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-06T22:47:24.004797+09:00", "mtime_epoch": 1783345644.0047965, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95504}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:47:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1558.695, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.8, "pushed": 0.8, "read": 0.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.154, "count": 14, "max_ms": 4.656, "p95_ms": 4.656}, "gstreamer_push": {"avg_ms": 1.288, "count": 14, "max_ms": 1.414, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.532, "count": 14, "max_ms": 85.108, "p95_ms": 85.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.129, "count": 14, "max_ms": 87.501, "p95_ms": 87.501}, "yolo11_640_fire_11class": {"avg_ms": 68.285, "count": 14, "max_ms": 86.47, "p95_ms": 86.47}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.501, "count": 14, "max_ms": 94.472, "p95_ms": 94.472}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.248, "count": 14, "max_ms": 107.385, "p95_ms": 107.385}, "read_interval": {"avg_ms": 2415.866, "count": 14, "max_ms": 16430.88, "p95_ms": 16430.88}, "triton_wait": {"avg_ms": 73.419, "count": 14, "max_ms": 94.484, "p95_ms": 94.484}, "visualize": {"avg_ms": 3.633, "count": 14, "max_ms": 3.852, "p95_ms": 3.852}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.5, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6073.289, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:47:24+09:00", "last_error_summary": "[h264 @ 0x55c91b825700] error while decoding MB 31 21, bytestream -5 [in#0/rtsp @ 0x55c91b022e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c91b022e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:47:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.808, "last_started_at": "2026-07-06T22:43:01+09:00", "pid": 33316, "process": {"exists": false, "pid": 33316}, "restart_count": 80, "restart_count_per_hour": 15, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.159, "sample_count": 2133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 306193, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:47:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T22:47:24+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-06T22:47:24+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T22:47:24.004797+09:00", "mtime_epoch": 1783345644.0047965, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95504}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:47:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1558.695, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.8, "pushed": 0.8, "read": 0.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.154, "count": 14, "max_ms": 4.656, "p95_ms": 4.656}, "gstreamer_push": {"avg_ms": 1.288, "count": 14, "max_ms": 1.414, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.532, "count": 14, "max_ms": 85.108, "p95_ms": 85.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.129, "count": 14, "max_ms": 87.501, "p95_ms": 87.501}, "yolo11_640_fire_11class": {"avg_ms": 68.285, "count": 14, "max_ms": 86.47, "p95_ms": 86.47}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.501, "count": 14, "max_ms": 94.472, "p95_ms": 94.472}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.248, "count": 14, "max_ms": 107.385, "p95_ms": 107.385}, "read_interval": {"avg_ms": 2415.866, "count": 14, "max_ms": 16430.88, "p95_ms": 16430.88}, "triton_wait": {"avg_ms": 73.419, "count": 14, "max_ms": 94.484, "p95_ms": 94.484}, "visualize": {"avg_ms": 3.633, "count": 14, "max_ms": 3.852, "p95_ms": 3.852}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.507, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6093.555, "limit_mb": 8796093022207.996}, "detail": {"pid": 33468, "restart_count": 81, "start_count": 82}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:47:24+09:00", "last_error_summary": "[h264 @ 0x55c91b825700] error while decoding MB 31 21, bytestream -5 [in#0/rtsp @ 0x55c91b022e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c91b022e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:47:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:47:26+09:00", "pid": 33468, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.062, "name": "ffmpeg", "pid": 33468, "rss_mb": 58.062, "state": "S (sleeping)", "threads": 1}, "restart_count": 81, "restart_count_per_hour": 16, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.896, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 306193, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:47:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.571, "last_frame_pushed_at": "2026-07-06T22:47:24+09:00", "last_frame_read_age_sec": 4.674, "last_frame_read_at": "2026-07-06T22:47:24+09:00", "mjpeg": {"age_sec": 4.555, "exists": true, "mtime": "2026-07-06T22:47:24.124798+09:00", "mtime_epoch": 1783345644.1247976, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94501}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:47:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1558.695, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.634, "pushed": 0.634, "read": 0.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.154, "count": 14, "max_ms": 4.656, "p95_ms": 4.656}, "gstreamer_push": {"avg_ms": 1.288, "count": 14, "max_ms": 1.414, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.532, "count": 14, "max_ms": 85.108, "p95_ms": 85.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.129, "count": 14, "max_ms": 87.501, "p95_ms": 87.501}, "yolo11_640_fire_11class": {"avg_ms": 68.285, "count": 14, "max_ms": 86.47, "p95_ms": 86.47}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.501, "count": 14, "max_ms": 94.472, "p95_ms": 94.472}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.248, "count": 14, "max_ms": 107.385, "p95_ms": 107.385}, "read_interval": {"avg_ms": 2415.866, "count": 14, "max_ms": 16430.88, "p95_ms": 16430.88}, "triton_wait": {"avg_ms": 73.419, "count": 14, "max_ms": 94.484, "p95_ms": 94.484}, "visualize": {"avg_ms": 3.633, "count": 14, "max_ms": 3.852, "p95_ms": 3.852}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.066, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2380}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6114.742, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56500b279c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500b279c40] error while decoding MB 0 28, bytestream 3494 [h264 @ 0x56500a9e5480] cabac decode of qscale diff failed at 7 29 [h264 @ 0x56500a9e5480] error while decoding MB 7 29, bytestream 4044 [in#0/rtsp @ 0x56500a8c2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500a8c2e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 408, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:48:22+09:00", "last_error_summary": "[h264 @ 0x56500b279c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500b279c40] error while decoding MB 0 28, bytestream 3494 [h264 @ 0x56500a9e5480] cabac decode of qscale diff failed at 7 29 [h264 @ 0x56500a9e5480] error while decoding MB 7 29, bytestream 4044 [in#0/rtsp @ 0x56500a8c2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500a8c2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:39:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 517.706, "last_started_at": "2026-07-06T22:39:44+09:00", "pid": 33149, "process": {"exists": false, "pid": 33149}, "restart_count": 132, "restart_count_per_hour": 6, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 133}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.441, "sample_count": 2599, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 285700, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:48:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T22:48:22+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-06T22:48:22+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-06T22:48:22.465321+09:00", "mtime_epoch": 1783345702.4653206, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118251}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:48:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1675.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.203, "pushed": 3.203, "read": 3.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.583, "count": 96, "max_ms": 5.7, "p95_ms": 4.534}, "gstreamer_push": {"avg_ms": 1.316, "count": 96, "max_ms": 2.31, "p95_ms": 1.5}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.566, "count": 96, "max_ms": 91.338, "p95_ms": 87.864}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.796, "count": 96, "max_ms": 113.829, "p95_ms": 101.361}, "yolo11_640_fire_11class": {"avg_ms": 75.504, "count": 96, "max_ms": 96.158, "p95_ms": 91.66}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.903, "count": 96, "max_ms": 103.974, "p95_ms": 99.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.571, "count": 96, "max_ms": 126.11, "p95_ms": 113.923}, "read_interval": {"avg_ms": 312.647, "count": 96, "max_ms": 19563.01, "p95_ms": 131.228}, "triton_wait": {"avg_ms": 84.069, "count": 96, "max_ms": 113.842, "p95_ms": 102.002}, "visualize": {"avg_ms": 3.73, "count": 96, "max_ms": 5.472, "p95_ms": 4.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2380}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6114.238, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:48:22+09:00", "last_error_summary": "[h264 @ 0x56500b279c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500b279c40] error while decoding MB 0 28, bytestream 3494 [h264 @ 0x56500a9e5480] cabac decode of qscale diff failed at 7 29 [h264 @ 0x56500a9e5480] error while decoding MB 7 29, bytestream 4044 [in#0/rtsp @ 0x56500a8c2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500a8c2e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 408, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:48:22+09:00", "last_error_summary": "[h264 @ 0x56500b279c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500b279c40] error while decoding MB 0 28, bytestream 3494 [h264 @ 0x56500a9e5480] cabac decode of qscale diff failed at 7 29 [h264 @ 0x56500a9e5480] error while decoding MB 7 29, bytestream 4044 [in#0/rtsp @ 0x56500a8c2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500a8c2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:48:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 517.789, "last_started_at": "2026-07-06T22:39:44+09:00", "pid": 33149, "process": {"exists": false, "pid": 33149}, "restart_count": 132, "restart_count_per_hour": 6, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 133}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.763, "sample_count": 2599, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 285701, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:48:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T22:48:22+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-06T22:48:22+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-06T22:48:22.465321+09:00", "mtime_epoch": 1783345702.4653206, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118251}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:48:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1675.434, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.208, "pushed": 3.208, "read": 3.174}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.583, "count": 96, "max_ms": 5.7, "p95_ms": 4.534}, "gstreamer_push": {"avg_ms": 1.319, "count": 96, "max_ms": 2.31, "p95_ms": 1.5}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.536, "count": 96, "max_ms": 91.338, "p95_ms": 87.864}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.147, "count": 96, "max_ms": 113.829, "p95_ms": 101.99}, "yolo11_640_fire_11class": {"avg_ms": 75.509, "count": 96, "max_ms": 96.158, "p95_ms": 91.66}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.889, "count": 96, "max_ms": 103.974, "p95_ms": 99.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.847, "count": 96, "max_ms": 126.11, "p95_ms": 116.295}, "read_interval": {"avg_ms": 314.716, "count": 95, "max_ms": 19563.01, "p95_ms": 131.228}, "triton_wait": {"avg_ms": 84.334, "count": 96, "max_ms": 113.842, "p95_ms": 103.985}, "visualize": {"avg_ms": 3.734, "count": 96, "max_ms": 5.472, "p95_ms": 4.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2380}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6114.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 408, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:48:22+09:00", "last_error_summary": "[h264 @ 0x56500b279c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500b279c40] error while decoding MB 0 28, bytestream 3494 [h264 @ 0x56500a9e5480] cabac decode of qscale diff failed at 7 29 [h264 @ 0x56500a9e5480] error while decoding MB 7 29, bytestream 4044 [in#0/rtsp @ 0x56500a8c2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500a8c2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:48:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 517.796, "last_started_at": "2026-07-06T22:39:44+09:00", "pid": 33149, "process": {"exists": false, "pid": 33149}, "restart_count": 132, "restart_count_per_hour": 6, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 133}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.763, "sample_count": 2599, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 285701, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:48:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-06T22:48:22+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-06T22:48:22+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-06T22:48:22.465321+09:00", "mtime_epoch": 1783345702.4653206, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118251}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:48:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1675.434, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.207, "pushed": 3.207, "read": 3.174}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.583, "count": 96, "max_ms": 5.7, "p95_ms": 4.534}, "gstreamer_push": {"avg_ms": 1.319, "count": 96, "max_ms": 2.31, "p95_ms": 1.5}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.536, "count": 96, "max_ms": 91.338, "p95_ms": 87.864}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.147, "count": 96, "max_ms": 113.829, "p95_ms": 101.99}, "yolo11_640_fire_11class": {"avg_ms": 75.509, "count": 96, "max_ms": 96.158, "p95_ms": 91.66}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.889, "count": 96, "max_ms": 103.974, "p95_ms": 99.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.847, "count": 96, "max_ms": 126.11, "p95_ms": 116.295}, "read_interval": {"avg_ms": 314.716, "count": 95, "max_ms": 19563.01, "p95_ms": 131.228}, "triton_wait": {"avg_ms": 84.334, "count": 96, "max_ms": 113.842, "p95_ms": 103.985}, "visualize": {"avg_ms": 3.734, "count": 96, "max_ms": 5.472, "p95_ms": 4.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2380}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6224.758, "limit_mb": 8796093022207.996}, "detail": {"pid": 33620, "restart_count": 133, "start_count": 134}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 408, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:48:22+09:00", "last_error_summary": "[h264 @ 0x56500b279c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500b279c40] error while decoding MB 0 28, bytestream 3494 [h264 @ 0x56500a9e5480] cabac decode of qscale diff failed at 7 29 [h264 @ 0x56500a9e5480] error while decoding MB 7 29, bytestream 4044 [in#0/rtsp @ 0x56500a8c2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500a8c2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:48:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:48:25+09:00", "pid": 33620, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.688, "name": "ffmpeg", "pid": 33620, "rss_mb": 141.688, "state": "S (sleeping)", "threads": 106}, "restart_count": 133, "restart_count_per_hour": 7, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 134}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.768, "sample_count": 2557, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 285701, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:48:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.527, "last_frame_pushed_at": "2026-07-06T22:48:22+09:00", "last_frame_read_age_sec": 4.653, "last_frame_read_at": "2026-07-06T22:48:22+09:00", "mjpeg": {"age_sec": 4.512, "exists": true, "mtime": "2026-07-06T22:48:22.601322+09:00", "mtime_epoch": 1783345702.6013217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116724}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:48:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1675.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.835, "pushed": 1.835, "read": 1.801}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.57, "count": 55, "max_ms": 5.7, "p95_ms": 4.128}, "gstreamer_push": {"avg_ms": 1.331, "count": 55, "max_ms": 2.31, "p95_ms": 1.452}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.032, "count": 55, "max_ms": 91.338, "p95_ms": 79.804}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.054, "count": 55, "max_ms": 113.829, "p95_ms": 105.395}, "yolo11_640_fire_11class": {"avg_ms": 75.751, "count": 55, "max_ms": 92.495, "p95_ms": 90.798}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.655, "count": 55, "max_ms": 100.547, "p95_ms": 99.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.753, "count": 55, "max_ms": 126.11, "p95_ms": 117.854}, "read_interval": {"avg_ms": 470.695, "count": 54, "max_ms": 19563.01, "p95_ms": 131.228}, "triton_wait": {"avg_ms": 85.227, "count": 55, "max_ms": 113.842, "p95_ms": 105.407}, "visualize": {"avg_ms": 3.741, "count": 55, "max_ms": 5.472, "p95_ms": 5.201}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6077.551, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ebeb7d0ec0] error while decoding MB 94 3, bytestream -37 [in#0/rtsp @ 0x55ebeaa26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ebeaa26e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:51:52+09:00", "last_error_summary": "[h264 @ 0x55ebeb7d0ec0] error while decoding MB 94 3, bytestream -37 [in#0/rtsp @ 0x55ebeaa26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ebeaa26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:47:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 266.262, "last_started_at": "2026-07-06T22:47:26+09:00", "pid": 33468, "process": {"exists": false, "pid": 33468}, "restart_count": 81, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.41, "sample_count": 2133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 308312, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:51:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T22:51:52+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T22:51:52+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T22:51:52.887475+09:00", "mtime_epoch": 1783345912.8874753, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101016}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:51:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1559.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 86, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.264, "pushed": 3.264, "read": 3.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.136, "count": 85, "max_ms": 5.174, "p95_ms": 4.868}, "gstreamer_push": {"avg_ms": 1.272, "count": 85, "max_ms": 1.418, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.33, "count": 85, "max_ms": 86.274, "p95_ms": 82.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.23, "count": 85, "max_ms": 115.18, "p95_ms": 106.281}, "yolo11_640_fire_11class": {"avg_ms": 76.422, "count": 85, "max_ms": 90.145, "p95_ms": 86.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.034, "count": 85, "max_ms": 101.421, "p95_ms": 97.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.533, "count": 85, "max_ms": 128.287, "p95_ms": 119.171}, "read_interval": {"avg_ms": 499.461, "count": 86, "max_ms": 16975.202, "p95_ms": 132.951}, "triton_wait": {"avg_ms": 86.701, "count": 85, "max_ms": 115.19, "p95_ms": 106.294}, "visualize": {"avg_ms": 3.615, "count": 85, "max_ms": 4.114, "p95_ms": 3.843}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.041, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6076.805, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:51:52+09:00", "last_error_summary": "[h264 @ 0x55ebeb7d0ec0] error while decoding MB 94 3, bytestream -37 [in#0/rtsp @ 0x55ebeaa26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ebeaa26e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:51:52+09:00", "last_error_summary": "[h264 @ 0x55ebeb7d0ec0] error while decoding MB 94 3, bytestream -37 [in#0/rtsp @ 0x55ebeaa26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ebeaa26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:51:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 266.301, "last_started_at": "2026-07-06T22:47:26+09:00", "pid": 33468, "process": {"exists": false, "pid": 33468}, "restart_count": 81, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.41, "sample_count": 2134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 308313, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:51:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T22:51:52+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-06T22:51:52+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-06T22:51:52.887475+09:00", "mtime_epoch": 1783345912.8874753, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101016}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:51:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1559.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 86, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.298, "pushed": 3.298, "read": 3.298}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.134, "count": 86, "max_ms": 5.174, "p95_ms": 4.868}, "gstreamer_push": {"avg_ms": 1.272, "count": 86, "max_ms": 1.418, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.429, "count": 86, "max_ms": 86.274, "p95_ms": 82.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.215, "count": 86, "max_ms": 115.18, "p95_ms": 106.281}, "yolo11_640_fire_11class": {"avg_ms": 76.449, "count": 86, "max_ms": 90.145, "p95_ms": 86.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.817, "count": 86, "max_ms": 101.421, "p95_ms": 97.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.454, "count": 86, "max_ms": 128.287, "p95_ms": 119.171}, "read_interval": {"avg_ms": 499.461, "count": 86, "max_ms": 16975.202, "p95_ms": 132.951}, "triton_wait": {"avg_ms": 86.623, "count": 86, "max_ms": 115.19, "p95_ms": 106.294}, "visualize": {"avg_ms": 3.616, "count": 86, "max_ms": 4.114, "p95_ms": 3.843}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.079, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6076.777, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:51:52+09:00", "last_error_summary": "[h264 @ 0x55ebeb7d0ec0] error while decoding MB 94 3, bytestream -37 [in#0/rtsp @ 0x55ebeaa26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ebeaa26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:51:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 266.308, "last_started_at": "2026-07-06T22:47:26+09:00", "pid": 33468, "process": {"exists": false, "pid": 33468}, "restart_count": 81, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.41, "sample_count": 2134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 308313, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:51:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T22:51:52+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T22:51:52+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-06T22:51:52.887475+09:00", "mtime_epoch": 1783345912.8874753, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101016}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:51:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1559.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 86, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.297, "pushed": 3.297, "read": 3.297}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.134, "count": 86, "max_ms": 5.174, "p95_ms": 4.868}, "gstreamer_push": {"avg_ms": 1.272, "count": 86, "max_ms": 1.418, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.429, "count": 86, "max_ms": 86.274, "p95_ms": 82.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.215, "count": 86, "max_ms": 115.18, "p95_ms": 106.281}, "yolo11_640_fire_11class": {"avg_ms": 76.449, "count": 86, "max_ms": 90.145, "p95_ms": 86.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.817, "count": 86, "max_ms": 101.421, "p95_ms": 97.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.454, "count": 86, "max_ms": 128.287, "p95_ms": 119.171}, "read_interval": {"avg_ms": 499.461, "count": 86, "max_ms": 16975.202, "p95_ms": 132.951}, "triton_wait": {"avg_ms": 86.623, "count": 86, "max_ms": 115.19, "p95_ms": 106.294}, "visualize": {"avg_ms": 3.616, "count": 86, "max_ms": 4.114, "p95_ms": 3.843}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.086, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6196.145, "limit_mb": 8796093022207.996}, "detail": {"pid": 33727, "restart_count": 82, "start_count": 83}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:51:52+09:00", "last_error_summary": "[h264 @ 0x55ebeb7d0ec0] error while decoding MB 94 3, bytestream -37 [in#0/rtsp @ 0x55ebeaa26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ebeaa26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:51:55+09:00", "pid": 33727, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.125, "name": "ffmpeg", "pid": 33727, "rss_mb": 149.125, "state": "R (running)", "threads": 49}, "restart_count": 82, "restart_count_per_hour": 15, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.41, "sample_count": 2134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 308313, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:51:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.104, "last_frame_pushed_at": "2026-07-06T22:51:52+09:00", "last_frame_read_age_sec": 4.199, "last_frame_read_at": "2026-07-06T22:51:52+09:00", "mjpeg": {"age_sec": 4.087, "exists": true, "mtime": "2026-07-06T22:51:52.991477+09:00", "mtime_epoch": 1783345912.9914765, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98917}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:51:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1559.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 85, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.305, "pushed": 6.305, "read": 6.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.135, "count": 85, "max_ms": 5.174, "p95_ms": 4.868}, "gstreamer_push": {"avg_ms": 1.272, "count": 85, "max_ms": 1.418, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.628, "count": 85, "max_ms": 86.274, "p95_ms": 82.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.504, "count": 85, "max_ms": 115.18, "p95_ms": 106.281}, "yolo11_640_fire_11class": {"avg_ms": 76.682, "count": 85, "max_ms": 90.145, "p95_ms": 86.263}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.16, "count": 85, "max_ms": 101.421, "p95_ms": 97.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.804, "count": 85, "max_ms": 128.287, "p95_ms": 119.171}, "read_interval": {"avg_ms": 305.629, "count": 85, "max_ms": 16696.235, "p95_ms": 130.236}, "triton_wait": {"avg_ms": 86.975, "count": 85, "max_ms": 115.19, "p95_ms": 106.294}, "visualize": {"avg_ms": 3.612, "count": 85, "max_ms": 4.114, "p95_ms": 3.811}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 13.481, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6120.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e2475bf280] cabac decode of qscale diff failed at 12 27 [h264 @ 0x55e2475bf280] error while decoding MB 12 27, bytestream 46266 [h264 @ 0x55e2475bf280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e2475bf280] error while decoding MB 0 21, bytestream 67247 [h264 @ 0x55e24760e880] error while decoding MB 44 27, bytestream -5 [h264 @ 0x55e24760e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e24760e880] error while decoding MB 0 7, bytestream ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 124, "h264_decode_error_count": 415, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:53:04+09:00", "last_error_summary": "[h264 @ 0x55e2475bf280] cabac decode of qscale diff failed at 12 27 [h264 @ 0x55e2475bf280] error while decoding MB 12 27, bytestream 46266 [h264 @ 0x55e2475bf280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e2475bf280] error while decoding MB 0 21, bytestream 67247 [h264 @ 0x55e24760e880] error while decoding MB 44 27, bytestream -5 [h264 @ 0x55e24760e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e24760e880] error while decoding MB 0 7, bytestream ", "last_exit_at": "2026-07-06T22:48:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 279.794, "last_started_at": "2026-07-06T22:48:25+09:00", "pid": 33620, "process": {"exists": false, "pid": 33620}, "restart_count": 133, "restart_count_per_hour": 6, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 134}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.661, "sample_count": 2399, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288089, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:53:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-06T22:53:04+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-06T22:53:04+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-06T22:53:04.840289+09:00", "mtime_epoch": 1783345984.8402894, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122047}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:53:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1676.465, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.268, "pushed": 3.268, "read": 3.268}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.767, "count": 98, "max_ms": 5.514, "p95_ms": 5.002}, "gstreamer_push": {"avg_ms": 1.333, "count": 98, "max_ms": 1.534, "p95_ms": 1.475}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.639, "count": 98, "max_ms": 90.806, "p95_ms": 83.34}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.684, "count": 98, "max_ms": 114.075, "p95_ms": 107.283}, "yolo11_640_fire_11class": {"avg_ms": 75.854, "count": 98, "max_ms": 92.273, "p95_ms": 86.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.248, "count": 98, "max_ms": 101.587, "p95_ms": 96.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.722, "count": 98, "max_ms": 128.208, "p95_ms": 121.553}, "read_interval": {"avg_ms": 306.467, "count": 98, "max_ms": 19132.228, "p95_ms": 140.741}, "triton_wait": {"avg_ms": 85.84, "count": 98, "max_ms": 114.09, "p95_ms": 107.295}, "visualize": {"avg_ms": 3.78, "count": 98, "max_ms": 4.989, "p95_ms": 4.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6120.043, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T22:53:04+09:00", "last_error_summary": "[h264 @ 0x55e2475bf280] cabac decode of qscale diff failed at 12 27 [h264 @ 0x55e2475bf280] error while decoding MB 12 27, bytestream 46266 [h264 @ 0x55e2475bf280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e2475bf280] error while decoding MB 0 21, bytestream 67247 [h264 @ 0x55e24760e880] error while decoding MB 44 27, bytestream -5 [h264 @ 0x55e24760e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e24760e880] error while decoding MB 0 7, bytestream ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 124, "h264_decode_error_count": 415, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:53:04+09:00", "last_error_summary": "[h264 @ 0x55e2475bf280] cabac decode of qscale diff failed at 12 27 [h264 @ 0x55e2475bf280] error while decoding MB 12 27, bytestream 46266 [h264 @ 0x55e2475bf280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e2475bf280] error while decoding MB 0 21, bytestream 67247 [h264 @ 0x55e24760e880] error while decoding MB 44 27, bytestream -5 [h264 @ 0x55e24760e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e24760e880] error while decoding MB 0 7, bytestream ", "last_exit_at": "2026-07-06T22:53:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 279.845, "last_started_at": "2026-07-06T22:48:25+09:00", "pid": 33620, "process": {"exists": false, "pid": 33620}, "restart_count": 133, "restart_count_per_hour": 6, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 134}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.661, "sample_count": 2400, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:53:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-06T22:53:04+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-06T22:53:04+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-06T22:53:04.840289+09:00", "mtime_epoch": 1783345984.8402894, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122047}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:53:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1676.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.275, "pushed": 3.275, "read": 3.242}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.766, "count": 98, "max_ms": 5.514, "p95_ms": 5.002}, "gstreamer_push": {"avg_ms": 1.333, "count": 98, "max_ms": 1.534, "p95_ms": 1.475}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.832, "count": 98, "max_ms": 90.951, "p95_ms": 85.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.576, "count": 98, "max_ms": 114.075, "p95_ms": 107.283}, "yolo11_640_fire_11class": {"avg_ms": 76.037, "count": 98, "max_ms": 92.273, "p95_ms": 87.793}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.364, "count": 98, "max_ms": 101.587, "p95_ms": 96.851}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.663, "count": 98, "max_ms": 128.208, "p95_ms": 121.553}, "read_interval": {"avg_ms": 308.294, "count": 97, "max_ms": 19132.228, "p95_ms": 140.741}, "triton_wait": {"avg_ms": 85.784, "count": 98, "max_ms": 114.09, "p95_ms": 107.295}, "visualize": {"avg_ms": 3.779, "count": 98, "max_ms": 4.989, "p95_ms": 4.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6120.055, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 124, "h264_decode_error_count": 415, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:53:04+09:00", "last_error_summary": "[h264 @ 0x55e2475bf280] cabac decode of qscale diff failed at 12 27 [h264 @ 0x55e2475bf280] error while decoding MB 12 27, bytestream 46266 [h264 @ 0x55e2475bf280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e2475bf280] error while decoding MB 0 21, bytestream 67247 [h264 @ 0x55e24760e880] error while decoding MB 44 27, bytestream -5 [h264 @ 0x55e24760e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e24760e880] error while decoding MB 0 7, bytestream ", "last_exit_at": "2026-07-06T22:53:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 279.853, "last_started_at": "2026-07-06T22:48:25+09:00", "pid": 33620, "process": {"exists": false, "pid": 33620}, "restart_count": 133, "restart_count_per_hour": 6, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 134}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.661, "sample_count": 2400, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:53:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-06T22:53:04+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-06T22:53:04+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-06T22:53:04.840289+09:00", "mtime_epoch": 1783345984.8402894, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122047}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:53:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1676.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.274, "pushed": 3.274, "read": 3.241}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.766, "count": 98, "max_ms": 5.514, "p95_ms": 5.002}, "gstreamer_push": {"avg_ms": 1.333, "count": 98, "max_ms": 1.534, "p95_ms": 1.475}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.832, "count": 98, "max_ms": 90.951, "p95_ms": 85.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.576, "count": 98, "max_ms": 114.075, "p95_ms": 107.283}, "yolo11_640_fire_11class": {"avg_ms": 76.037, "count": 98, "max_ms": 92.273, "p95_ms": 87.793}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.364, "count": 98, "max_ms": 101.587, "p95_ms": 96.851}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.663, "count": 98, "max_ms": 128.208, "p95_ms": 121.553}, "read_interval": {"avg_ms": 308.294, "count": 97, "max_ms": 19132.228, "p95_ms": 140.741}, "triton_wait": {"avg_ms": 85.784, "count": 98, "max_ms": 114.09, "p95_ms": 107.295}, "visualize": {"avg_ms": 3.779, "count": 98, "max_ms": 4.989, "p95_ms": 4.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6227.828, "limit_mb": 8796093022207.996}, "detail": {"pid": 33879, "restart_count": 134, "start_count": 135}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 124, "h264_decode_error_count": 415, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:53:04+09:00", "last_error_summary": "[h264 @ 0x55e2475bf280] cabac decode of qscale diff failed at 12 27 [h264 @ 0x55e2475bf280] error while decoding MB 12 27, bytestream 46266 [h264 @ 0x55e2475bf280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e2475bf280] error while decoding MB 0 21, bytestream 67247 [h264 @ 0x55e24760e880] error while decoding MB 44 27, bytestream -5 [h264 @ 0x55e24760e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e24760e880] error while decoding MB 0 7, bytestream ", "last_exit_at": "2026-07-06T22:53:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:53:07+09:00", "pid": 33879, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.074, "name": "ffmpeg", "pid": 33879, "rss_mb": 139.074, "state": "S (sleeping)", "threads": 106}, "restart_count": 134, "restart_count_per_hour": 7, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 135}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.661, "sample_count": 2400, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:53:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.406, "last_frame_pushed_at": "2026-07-06T22:53:04+09:00", "last_frame_read_age_sec": 4.517, "last_frame_read_at": "2026-07-06T22:53:04+09:00", "mjpeg": {"age_sec": 4.392, "exists": true, "mtime": "2026-07-06T22:53:04.960291+09:00", "mtime_epoch": 1783345984.960291, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T22:53:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1676.465, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.969, "pushed": 1.969, "read": 1.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.627, "count": 59, "max_ms": 5.514, "p95_ms": 4.738}, "gstreamer_push": {"avg_ms": 1.321, "count": 59, "max_ms": 1.534, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.05, "count": 59, "max_ms": 90.951, "p95_ms": 83.34}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.699, "count": 59, "max_ms": 111.103, "p95_ms": 107.283}, "yolo11_640_fire_11class": {"avg_ms": 76.687, "count": 59, "max_ms": 92.273, "p95_ms": 90.139}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.136, "count": 59, "max_ms": 98.594, "p95_ms": 96.867}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.08, "count": 59, "max_ms": 124.733, "p95_ms": 121.553}, "read_interval": {"avg_ms": 440.445, "count": 58, "max_ms": 19132.228, "p95_ms": 140.741}, "triton_wait": {"avg_ms": 86.451, "count": 59, "max_ms": 111.116, "p95_ms": 107.295}, "visualize": {"avg_ms": 3.74, "count": 59, "max_ms": 4.989, "p95_ms": 4.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6349.68, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:51:52+09:00", "last_error_summary": "[h264 @ 0x55ebeb7d0ec0] error while decoding MB 94 3, bytestream -37 [in#0/rtsp @ 0x55ebeaa26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ebeaa26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 182.015, "last_started_at": "2026-07-06T22:51:55+09:00", "pid": 33727, "process": {"exists": true, "fd_count": 5, "hwm_mb": 328.906, "name": "ffmpeg", "pid": 33727, "rss_mb": 328.906, "state": "S (sleeping)", "threads": 106}, "restart_count": 82, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.26, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 309694, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:54:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.061, "last_frame_pushed_at": "2026-07-06T22:54:27+09:00", "last_frame_read_age_sec": 30.157, "last_frame_read_at": "2026-07-06T22:54:26+09:00", "mjpeg": {"age_sec": 30.043, "exists": true, "mtime": "2026-07-06T22:54:27.049258+09:00", "mtime_epoch": 1783346067.0492575, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96296}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:54:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1560.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6337.602, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:51:52+09:00", "last_error_summary": "[h264 @ 0x55ebeb7d0ec0] error while decoding MB 94 3, bytestream -37 [in#0/rtsp @ 0x55ebeaa26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ebeaa26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 182.027, "last_started_at": "2026-07-06T22:51:55+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 82, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.26, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 309694, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:54:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.073, "last_frame_pushed_at": "2026-07-06T22:54:27+09:00", "last_frame_read_age_sec": 30.169, "last_frame_read_at": "2026-07-06T22:54:26+09:00", "mjpeg": {"age_sec": 30.055, "exists": true, "mtime": "2026-07-06T22:54:27.049258+09:00", "mtime_epoch": 1783346067.0492575, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96296}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:54:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1560.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6054.305, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560f233d7c00] error while decoding MB 94 46, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:54:57+09:00", "last_error_summary": "[h264 @ 0x560f233d7c00] error while decoding MB 94 46, bytestream -7", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 182.079, "last_started_at": "2026-07-06T22:51:55+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 82, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.26, "sample_count": 2117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 309694, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:54:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.125, "last_frame_pushed_at": "2026-07-06T22:54:27+09:00", "last_frame_read_age_sec": 30.222, "last_frame_read_at": "2026-07-06T22:54:26+09:00", "mjpeg": {"age_sec": 30.108, "exists": true, "mtime": "2026-07-06T22:54:27.049258+09:00", "mtime_epoch": 1783346067.0492575, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96296}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:54:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1560.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6081.402, "limit_mb": 8796093022207.996}, "detail": {"pid": 33986, "restart_count": 83, "start_count": 84}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:54:57+09:00", "last_error_summary": "[h264 @ 0x560f233d7c00] error while decoding MB 94 46, bytestream -7", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:54:59+09:00", "pid": 33986, "process": {"exists": true, "fd_count": 5, "hwm_mb": 63.695, "name": "ffmpeg", "pid": 33986, "rss_mb": 63.695, "state": "S (sleeping)", "threads": 19}, "restart_count": 83, "restart_count_per_hour": 15, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.896, "sample_count": 2075, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 309694, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:54:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.909, "last_frame_pushed_at": "2026-07-06T22:54:27+09:00", "last_frame_read_age_sec": 35.006, "last_frame_read_at": "2026-07-06T22:54:26+09:00", "mjpeg": {"age_sec": 34.892, "exists": true, "mtime": "2026-07-06T22:54:27.049258+09:00", "mtime_epoch": 1783346067.0492575, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96296}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:55:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1560.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6375.527, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:54:57+09:00", "last_error_summary": "[h264 @ 0x560f233d7c00] error while decoding MB 94 46, bytestream -7", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.725, "last_started_at": "2026-07-06T22:54:59+09:00", "pid": 33986, "process": {"exists": true, "fd_count": 5, "hwm_mb": 329.957, "name": "ffmpeg", "pid": 33986, "rss_mb": 329.957, "state": "S (sleeping)", "threads": 106}, "restart_count": 83, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.133, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 310544, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:56:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.058, "last_frame_pushed_at": "2026-07-06T22:56:37+09:00", "last_frame_read_age_sec": 30.163, "last_frame_read_at": "2026-07-06T22:56:37+09:00", "mjpeg": {"age_sec": 30.038, "exists": true, "mtime": "2026-07-06T22:56:37.626864+09:00", "mtime_epoch": 1783346197.6268642, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100257}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:57:07+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1560.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6367.504, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:54:57+09:00", "last_error_summary": "[h264 @ 0x560f233d7c00] error while decoding MB 94 46, bytestream -7", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.738, "last_started_at": "2026-07-06T22:54:59+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 83, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.133, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 310544, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:56:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.07, "last_frame_pushed_at": "2026-07-06T22:56:37+09:00", "last_frame_read_age_sec": 30.176, "last_frame_read_at": "2026-07-06T22:56:37+09:00", "mjpeg": {"age_sec": 30.051, "exists": true, "mtime": "2026-07-06T22:56:37.626864+09:00", "mtime_epoch": 1783346197.6268642, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100257}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:57:07+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1560.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6074.676, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fd50741b40] error while decoding MB 84 17, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:57:07+09:00", "last_error_summary": "[h264 @ 0x55fd50741b40] error while decoding MB 84 17, bytestream -5", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.789, "last_started_at": "2026-07-06T22:54:59+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 83, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.133, "sample_count": 2136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 310544, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:56:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.121, "last_frame_pushed_at": "2026-07-06T22:56:37+09:00", "last_frame_read_age_sec": 30.227, "last_frame_read_at": "2026-07-06T22:56:37+09:00", "mjpeg": {"age_sec": 30.102, "exists": true, "mtime": "2026-07-06T22:56:37.626864+09:00", "mtime_epoch": 1783346197.6268642, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100257}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:57:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1560.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6256.527, "limit_mb": 8796093022207.996}, "detail": {"pid": 34138, "restart_count": 84, "start_count": 85}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:57:07+09:00", "last_error_summary": "[h264 @ 0x55fd50741b40] error while decoding MB 84 17, bytestream -5", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:57:10+09:00", "pid": 34138, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.477, "name": "ffmpeg", "pid": 34138, "rss_mb": 209.477, "state": "S (sleeping)", "threads": 106}, "restart_count": 84, "restart_count_per_hour": 15, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.412, "sample_count": 2093, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 310544, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:56:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.955, "last_frame_pushed_at": "2026-07-06T22:56:37+09:00", "last_frame_read_age_sec": 35.061, "last_frame_read_at": "2026-07-06T22:56:37+09:00", "mjpeg": {"age_sec": 34.936, "exists": true, "mtime": "2026-07-06T22:56:37.626864+09:00", "mtime_epoch": 1783346197.6268642, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100257}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:57:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1560.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2593}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6381.004, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:57:07+09:00", "last_error_summary": "[h264 @ 0x55fd50741b40] error while decoding MB 84 17, bytestream -5", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.772, "last_started_at": "2026-07-06T22:57:10+09:00", "pid": 34138, "process": {"exists": true, "fd_count": 5, "hwm_mb": 328.316, "name": "ffmpeg", "pid": 34138, "rss_mb": 328.316, "state": "S (sleeping)", "threads": 106}, "restart_count": 84, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.158, "sample_count": 1776, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 311278, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:58:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.056, "last_frame_pushed_at": "2026-07-06T22:58:35+09:00", "last_frame_read_age_sec": 30.158, "last_frame_read_at": "2026-07-06T22:58:35+09:00", "mjpeg": {"age_sec": 30.041, "exists": true, "mtime": "2026-07-06T22:58:35.292371+09:00", "mtime_epoch": 1783346315.2923708, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:59:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1561.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2593}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6372.949, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:57:07+09:00", "last_error_summary": "[h264 @ 0x55fd50741b40] error while decoding MB 84 17, bytestream -5", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.784, "last_started_at": "2026-07-06T22:57:10+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 84, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.158, "sample_count": 1776, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 311278, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:58:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.068, "last_frame_pushed_at": "2026-07-06T22:58:35+09:00", "last_frame_read_age_sec": 30.17, "last_frame_read_at": "2026-07-06T22:58:35+09:00", "mjpeg": {"age_sec": 30.053, "exists": true, "mtime": "2026-07-06T22:58:35.292371+09:00", "mtime_epoch": 1783346315.2923708, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:59:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1561.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2593}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6080.223, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5636002a9180] error while decoding MB 50 18, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:59:05+09:00", "last_error_summary": "[h264 @ 0x5636002a9180] error while decoding MB 50 18, bytestream -5", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.834, "last_started_at": "2026-07-06T22:57:10+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 84, "restart_count_per_hour": 14, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.158, "sample_count": 1776, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 311278, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:58:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.118, "last_frame_pushed_at": "2026-07-06T22:58:35+09:00", "last_frame_read_age_sec": 30.22, "last_frame_read_at": "2026-07-06T22:58:35+09:00", "mjpeg": {"age_sec": 30.103, "exists": true, "mtime": "2026-07-06T22:58:35.292371+09:00", "mtime_epoch": 1783346315.2923708, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:59:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1561.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2593}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6195.512, "limit_mb": 8796093022207.996}, "detail": {"pid": 34290, "restart_count": 85, "start_count": 86}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T22:59:05+09:00", "last_error_summary": "[h264 @ 0x5636002a9180] error while decoding MB 50 18, bytestream -5", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T22:59:08+09:00", "pid": 34290, "process": {"exists": true, "fd_count": 5, "hwm_mb": 148.266, "name": "ffmpeg", "pid": 34290, "rss_mb": 148.266, "state": "R (running)", "threads": 49}, "restart_count": 85, "restart_count_per_hour": 15, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.177, "sample_count": 1735, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 311278, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T22:58:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.792, "last_frame_pushed_at": "2026-07-06T22:58:35+09:00", "last_frame_read_age_sec": 34.894, "last_frame_read_at": "2026-07-06T22:58:35+09:00", "mjpeg": {"age_sec": 34.777, "exists": true, "mtime": "2026-07-06T22:58:35.292371+09:00", "mtime_epoch": 1783346315.2923708, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T22:59:10+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1561.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2433}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6125.168, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d5e95ad900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e95ad900] error while decoding MB 0 29, bytestream 1919 [h264 @ 0x55d5ea154600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5ea154600] error while decoding MB 0 9, bytestream 6674 [h264 @ 0x55d5e9fb6900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e9fb6900] error while decoding MB 0 32, bytestream 4336 [h264 @ 0x55d5e9ee7b40] left block unavailable for reques", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 420, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:00:26+09:00", "last_error_summary": "[h264 @ 0x55d5e95ad900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e95ad900] error while decoding MB 0 29, bytestream 1919 [h264 @ 0x55d5ea154600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5ea154600] error while decoding MB 0 9, bytestream 6674 [h264 @ 0x55d5e9fb6900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e9fb6900] error while decoding MB 0 32, bytestream 4336 [h264 @ 0x55d5e9ee7b40] left block unavailable for reques", "last_exit_at": "2026-07-06T22:53:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 439.057, "last_started_at": "2026-07-06T22:53:07+09:00", "pid": 33879, "process": {"exists": false, "pid": 33879}, "restart_count": 134, "restart_count_per_hour": 6, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 135}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.333, "sample_count": 2662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292049, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:00:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.051, "last_frame_pushed_at": "2026-07-06T23:00:26+09:00", "last_frame_read_age_sec": 0.042, "last_frame_read_at": "2026-07-06T23:00:26+09:00", "mjpeg": {"age_sec": 0.03, "exists": true, "mtime": "2026-07-06T23:00:26.377833+09:00", "mtime_epoch": 1783346426.3778334, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:00:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1678.27, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.601, "count": 99, "max_ms": 4.973, "p95_ms": 4.305}, "gstreamer_push": {"avg_ms": 1.34, "count": 99, "max_ms": 1.661, "p95_ms": 1.501}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.97, "count": 99, "max_ms": 87.606, "p95_ms": 81.491}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.507, "count": 99, "max_ms": 119.145, "p95_ms": 93.108}, "yolo11_640_fire_11class": {"avg_ms": 73.311, "count": 99, "max_ms": 86.894, "p95_ms": 84.921}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.157, "count": 99, "max_ms": 99.984, "p95_ms": 96.297}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.343, "count": 99, "max_ms": 133.662, "p95_ms": 109.134}, "read_interval": {"avg_ms": 300.714, "count": 100, "max_ms": 10124.219, "p95_ms": 125.971}, "triton_wait": {"avg_ms": 83.93, "count": 99, "max_ms": 119.158, "p95_ms": 96.741}, "visualize": {"avg_ms": 3.683, "count": 99, "max_ms": 4.812, "p95_ms": 4.405}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2433}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6124.469, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:00:26+09:00", "last_error_summary": "[h264 @ 0x55d5e95ad900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e95ad900] error while decoding MB 0 29, bytestream 1919 [h264 @ 0x55d5ea154600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5ea154600] error while decoding MB 0 9, bytestream 6674 [h264 @ 0x55d5e9fb6900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e9fb6900] error while decoding MB 0 32, bytestream 4336 [h264 @ 0x55d5e9ee7b40] left block unavailable for reques", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 420, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:00:26+09:00", "last_error_summary": "[h264 @ 0x55d5e95ad900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e95ad900] error while decoding MB 0 29, bytestream 1919 [h264 @ 0x55d5ea154600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5ea154600] error while decoding MB 0 9, bytestream 6674 [h264 @ 0x55d5e9fb6900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e9fb6900] error while decoding MB 0 32, bytestream 4336 [h264 @ 0x55d5e9ee7b40] left block unavailable for reques", "last_exit_at": "2026-07-06T23:00:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 439.133, "last_started_at": "2026-07-06T22:53:07+09:00", "pid": 33879, "process": {"exists": false, "pid": 33879}, "restart_count": 134, "restart_count_per_hour": 6, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 135}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.333, "sample_count": 2663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:00:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T23:00:26+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T23:00:26+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T23:00:26.377833+09:00", "mtime_epoch": 1783346426.3778334, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:00:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1678.27, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.337, "pushed": 3.337, "read": 3.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.6, "count": 100, "max_ms": 4.973, "p95_ms": 4.255}, "gstreamer_push": {"avg_ms": 1.34, "count": 100, "max_ms": 1.661, "p95_ms": 1.488}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.133, "count": 100, "max_ms": 87.606, "p95_ms": 81.491}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.646, "count": 100, "max_ms": 119.145, "p95_ms": 91.413}, "yolo11_640_fire_11class": {"avg_ms": 73.461, "count": 100, "max_ms": 88.282, "p95_ms": 84.921}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.292, "count": 100, "max_ms": 99.984, "p95_ms": 96.297}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.469, "count": 100, "max_ms": 133.662, "p95_ms": 108.931}, "read_interval": {"avg_ms": 302.542, "count": 99, "max_ms": 10124.219, "p95_ms": 126.215}, "triton_wait": {"avg_ms": 84.058, "count": 100, "max_ms": 119.158, "p95_ms": 96.741}, "visualize": {"avg_ms": 3.683, "count": 100, "max_ms": 4.812, "p95_ms": 4.271}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2433}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6124.621, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 420, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:00:26+09:00", "last_error_summary": "[h264 @ 0x55d5e95ad900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e95ad900] error while decoding MB 0 29, bytestream 1919 [h264 @ 0x55d5ea154600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5ea154600] error while decoding MB 0 9, bytestream 6674 [h264 @ 0x55d5e9fb6900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e9fb6900] error while decoding MB 0 32, bytestream 4336 [h264 @ 0x55d5e9ee7b40] left block unavailable for reques", "last_exit_at": "2026-07-06T23:00:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 439.14, "last_started_at": "2026-07-06T22:53:07+09:00", "pid": 33879, "process": {"exists": false, "pid": 33879}, "restart_count": 134, "restart_count_per_hour": 6, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 135}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.333, "sample_count": 2663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:00:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T23:00:26+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T23:00:26+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T23:00:26.377833+09:00", "mtime_epoch": 1783346426.3778334, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:00:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1678.27, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.336, "pushed": 3.336, "read": 3.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.6, "count": 100, "max_ms": 4.973, "p95_ms": 4.255}, "gstreamer_push": {"avg_ms": 1.34, "count": 100, "max_ms": 1.661, "p95_ms": 1.488}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.133, "count": 100, "max_ms": 87.606, "p95_ms": 81.491}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.646, "count": 100, "max_ms": 119.145, "p95_ms": 91.413}, "yolo11_640_fire_11class": {"avg_ms": 73.461, "count": 100, "max_ms": 88.282, "p95_ms": 84.921}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.292, "count": 100, "max_ms": 99.984, "p95_ms": 96.297}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.469, "count": 100, "max_ms": 133.662, "p95_ms": 108.931}, "read_interval": {"avg_ms": 302.542, "count": 99, "max_ms": 10124.219, "p95_ms": 126.215}, "triton_wait": {"avg_ms": 84.058, "count": 100, "max_ms": 119.158, "p95_ms": 96.741}, "visualize": {"avg_ms": 3.683, "count": 100, "max_ms": 4.812, "p95_ms": 4.271}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2433}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6230.867, "limit_mb": 8796093022207.996}, "detail": {"pid": 34442, "restart_count": 135, "start_count": 136}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 420, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:00:26+09:00", "last_error_summary": "[h264 @ 0x55d5e95ad900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e95ad900] error while decoding MB 0 29, bytestream 1919 [h264 @ 0x55d5ea154600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5ea154600] error while decoding MB 0 9, bytestream 6674 [h264 @ 0x55d5e9fb6900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d5e9fb6900] error while decoding MB 0 32, bytestream 4336 [h264 @ 0x55d5e9ee7b40] left block unavailable for reques", "last_exit_at": "2026-07-06T23:00:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:00:29+09:00", "pid": 34442, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.625, "name": "ffmpeg", "pid": 34442, "rss_mb": 139.625, "state": "S (sleeping)", "threads": 106}, "restart_count": 135, "restart_count_per_hour": 7, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 136}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.243, "sample_count": 2619, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:00:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.975, "last_frame_pushed_at": "2026-07-06T23:00:26+09:00", "last_frame_read_age_sec": 5.086, "last_frame_read_at": "2026-07-06T23:00:26+09:00", "mjpeg": {"age_sec": 4.959, "exists": true, "mtime": "2026-07-06T23:00:26.493835+09:00", "mtime_epoch": 1783346426.493835, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123181}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:00:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1678.27, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.835, "pushed": 1.835, "read": 1.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.605, "count": 55, "max_ms": 4.973, "p95_ms": 4.255}, "gstreamer_push": {"avg_ms": 1.344, "count": 55, "max_ms": 1.661, "p95_ms": 1.506}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.211, "count": 55, "max_ms": 87.281, "p95_ms": 81.031}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.591, "count": 55, "max_ms": 101.218, "p95_ms": 93.108}, "yolo11_640_fire_11class": {"avg_ms": 74.917, "count": 55, "max_ms": 88.282, "p95_ms": 85.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.326, "count": 55, "max_ms": 99.984, "p95_ms": 96.731}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.215, "count": 55, "max_ms": 115.495, "p95_ms": 109.415}, "read_interval": {"avg_ms": 462.914, "count": 54, "max_ms": 10124.219, "p95_ms": 130.882}, "triton_wait": {"avg_ms": 84.797, "count": 55, "max_ms": 101.232, "p95_ms": 97.106}, "visualize": {"avg_ms": 3.688, "count": 55, "max_ms": 4.536, "p95_ms": 3.87}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6131.777, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5613f4ebaa80] error while decoding MB 6 29, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 126, "h264_decode_error_count": 421, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:08:01+09:00", "last_error_summary": "[h264 @ 0x5613f4ebaa80] error while decoding MB 6 29, bytestream -7", "last_exit_at": "2026-07-06T23:00:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 452.208, "last_started_at": "2026-07-06T23:00:29+09:00", "pid": 34442, "process": {"exists": false, "pid": 34442}, "restart_count": 135, "restart_count_per_hour": 7, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 136}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.53, "sample_count": 2674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 296062, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:08:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-06T23:08:01+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T23:08:01+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-06T23:08:01.612095+09:00", "mtime_epoch": 1783346881.612095, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123396}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:08:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1680.332, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 274, "pushed_frames": 274, "read_frames": 274, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.139, "pushed": 9.139, "read": 9.139}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.68, "count": 274, "max_ms": 5.42, "p95_ms": 5.126}, "gstreamer_push": {"avg_ms": 1.292, "count": 274, "max_ms": 1.52, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.456, "count": 274, "max_ms": 88.402, "p95_ms": 82.225}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.117, "count": 274, "max_ms": 110.394, "p95_ms": 100.083}, "yolo11_640_fire_11class": {"avg_ms": 74.827, "count": 274, "max_ms": 89.98, "p95_ms": 85.986}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.039, "count": 274, "max_ms": 100.374, "p95_ms": 95.356}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.381, "count": 274, "max_ms": 122.883, "p95_ms": 112.97}, "read_interval": {"avg_ms": 109.593, "count": 274, "max_ms": 136.822, "p95_ms": 126.367}, "triton_wait": {"avg_ms": 84.065, "count": 274, "max_ms": 110.406, "p95_ms": 100.387}, "visualize": {"avg_ms": 3.588, "count": 274, "max_ms": 5.047, "p95_ms": 4.249}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6130.73, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:08:01+09:00", "last_error_summary": "[h264 @ 0x5613f4ebaa80] error while decoding MB 6 29, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 126, "h264_decode_error_count": 421, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:08:01+09:00", "last_error_summary": "[h264 @ 0x5613f4ebaa80] error while decoding MB 6 29, bytestream -7", "last_exit_at": "2026-07-06T23:08:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 452.264, "last_started_at": "2026-07-06T23:00:29+09:00", "pid": 34442, "process": {"exists": false, "pid": 34442}, "restart_count": 135, "restart_count_per_hour": 7, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 136}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.53, "sample_count": 2675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 296063, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:08:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T23:08:01+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-06T23:08:01+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-06T23:08:01.612095+09:00", "mtime_epoch": 1783346881.612095, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123396}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:08:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1680.332, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 274, "pushed_frames": 274, "read_frames": 273, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.159, "pushed": 9.159, "read": 9.126}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.679, "count": 274, "max_ms": 5.42, "p95_ms": 5.126}, "gstreamer_push": {"avg_ms": 1.292, "count": 274, "max_ms": 1.52, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.438, "count": 274, "max_ms": 88.402, "p95_ms": 82.225}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.119, "count": 274, "max_ms": 110.394, "p95_ms": 100.083}, "yolo11_640_fire_11class": {"avg_ms": 74.827, "count": 274, "max_ms": 89.98, "p95_ms": 85.986}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.04, "count": 274, "max_ms": 100.374, "p95_ms": 95.356}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.38, "count": 274, "max_ms": 122.883, "p95_ms": 112.97}, "read_interval": {"avg_ms": 109.587, "count": 273, "max_ms": 136.822, "p95_ms": 126.367}, "triton_wait": {"avg_ms": 84.066, "count": 274, "max_ms": 110.406, "p95_ms": 100.387}, "visualize": {"avg_ms": 3.588, "count": 274, "max_ms": 5.047, "p95_ms": 4.249}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6130.703, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 126, "h264_decode_error_count": 421, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:08:01+09:00", "last_error_summary": "[h264 @ 0x5613f4ebaa80] error while decoding MB 6 29, bytestream -7", "last_exit_at": "2026-07-06T23:08:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 452.274, "last_started_at": "2026-07-06T23:00:29+09:00", "pid": 34442, "process": {"exists": false, "pid": 34442}, "restart_count": 135, "restart_count_per_hour": 7, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 136}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.53, "sample_count": 2675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 296063, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:08:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T23:08:01+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-06T23:08:01+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T23:08:01.612095+09:00", "mtime_epoch": 1783346881.612095, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123396}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:08:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1680.332, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 274, "pushed_frames": 274, "read_frames": 273, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.156, "pushed": 9.156, "read": 9.123}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.679, "count": 274, "max_ms": 5.42, "p95_ms": 5.126}, "gstreamer_push": {"avg_ms": 1.292, "count": 274, "max_ms": 1.52, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.438, "count": 274, "max_ms": 88.402, "p95_ms": 82.225}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.119, "count": 274, "max_ms": 110.394, "p95_ms": 100.083}, "yolo11_640_fire_11class": {"avg_ms": 74.827, "count": 274, "max_ms": 89.98, "p95_ms": 85.986}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.04, "count": 274, "max_ms": 100.374, "p95_ms": 95.356}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.38, "count": 274, "max_ms": 122.883, "p95_ms": 112.97}, "read_interval": {"avg_ms": 109.587, "count": 273, "max_ms": 136.822, "p95_ms": 126.367}, "triton_wait": {"avg_ms": 84.066, "count": 274, "max_ms": 110.406, "p95_ms": 100.387}, "visualize": {"avg_ms": 3.588, "count": 274, "max_ms": 5.047, "p95_ms": 4.249}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6241.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 34549, "restart_count": 136, "start_count": 137}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 126, "h264_decode_error_count": 421, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:08:01+09:00", "last_error_summary": "[h264 @ 0x5613f4ebaa80] error while decoding MB 6 29, bytestream -7", "last_exit_at": "2026-07-06T23:08:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:08:03+09:00", "pid": 34549, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.762, "name": "ffmpeg", "pid": 34549, "rss_mb": 141.762, "state": "S (sleeping)", "threads": 106}, "restart_count": 136, "restart_count_per_hour": 8, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 137}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.525, "sample_count": 2638, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 296063, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:08:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.119, "last_frame_pushed_at": "2026-07-06T23:08:01+09:00", "last_frame_read_age_sec": 4.22, "last_frame_read_at": "2026-07-06T23:08:01+09:00", "mjpeg": {"age_sec": 4.104, "exists": true, "mtime": "2026-07-06T23:08:01.724097+09:00", "mtime_epoch": 1783346881.7240968, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:08:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1680.332, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 237, "pushed_frames": 237, "read_frames": 236, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.903, "pushed": 7.903, "read": 7.87}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.531, "count": 237, "max_ms": 5.322, "p95_ms": 4.979}, "gstreamer_push": {"avg_ms": 1.29, "count": 237, "max_ms": 1.52, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.359, "count": 237, "max_ms": 88.402, "p95_ms": 82.225}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.558, "count": 237, "max_ms": 110.394, "p95_ms": 100.803}, "yolo11_640_fire_11class": {"avg_ms": 75.008, "count": 237, "max_ms": 89.98, "p95_ms": 85.986}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.205, "count": 237, "max_ms": 100.374, "p95_ms": 94.251}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.458, "count": 237, "max_ms": 122.883, "p95_ms": 113.125}, "read_interval": {"avg_ms": 109.605, "count": 236, "max_ms": 136.822, "p95_ms": 126.667}, "triton_wait": {"avg_ms": 84.36, "count": 237, "max_ms": 110.406, "p95_ms": 100.815}, "visualize": {"avg_ms": 3.547, "count": 237, "max_ms": 5.047, "p95_ms": 3.988}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6136.711, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5589eb6c15c0] reference picture missing during reorder [h264 @ 0x5589eb6c15c0] Missing reference picture, default is 65562 [h264 @ 0x5589eb6d56c0] mmco: unref short failure [h264 @ 0x5589eb6d56c0] illegal short term buffer state detected [h264 @ 0x5589eb714b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb714b80] error while decoding MB 0 18, bytestream 8193 [h264 @ 0x5589eb7f1100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb7f1100] er", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 127, "h264_decode_error_count": 425, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:14:33+09:00", "last_error_summary": "[h264 @ 0x5589eb6c15c0] reference picture missing during reorder [h264 @ 0x5589eb6c15c0] Missing reference picture, default is 65562 [h264 @ 0x5589eb6d56c0] mmco: unref short failure [h264 @ 0x5589eb6d56c0] illegal short term buffer state detected [h264 @ 0x5589eb714b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb714b80] error while decoding MB 0 18, bytestream 8193 [h264 @ 0x5589eb7f1100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb7f1100] er", "last_exit_at": "2026-07-06T23:08:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 389.244, "last_started_at": "2026-07-06T23:08:03+09:00", "pid": 34549, "process": {"exists": false, "pid": 34549}, "restart_count": 136, "restart_count_per_hour": 7, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 137}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.412, "sample_count": 2509, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299358, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:14:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-06T23:14:32+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T23:14:33+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-06T23:14:33.009669+09:00", "mtime_epoch": 1783347273.009669, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124258}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:14:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1681.879, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 86, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.834, "pushed": 2.834, "read": 2.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.713, "count": 85, "max_ms": 5.416, "p95_ms": 5.012}, "gstreamer_push": {"avg_ms": 1.314, "count": 85, "max_ms": 1.538, "p95_ms": 1.432}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.315, "count": 85, "max_ms": 89.062, "p95_ms": 86.347}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.855, "count": 85, "max_ms": 115.964, "p95_ms": 107.039}, "yolo11_640_fire_11class": {"avg_ms": 76.761, "count": 85, "max_ms": 89.044, "p95_ms": 87.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.792, "count": 85, "max_ms": 101.193, "p95_ms": 95.696}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.608, "count": 85, "max_ms": 128.021, "p95_ms": 119.384}, "read_interval": {"avg_ms": 349.305, "count": 86, "max_ms": 20544.316, "p95_ms": 132.608}, "triton_wait": {"avg_ms": 86.159, "count": 85, "max_ms": 115.978, "p95_ms": 107.051}, "visualize": {"avg_ms": 3.645, "count": 85, "max_ms": 4.219, "p95_ms": 3.925}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6136.191, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:14:33+09:00", "last_error_summary": "[h264 @ 0x5589eb6c15c0] reference picture missing during reorder [h264 @ 0x5589eb6c15c0] Missing reference picture, default is 65562 [h264 @ 0x5589eb6d56c0] mmco: unref short failure [h264 @ 0x5589eb6d56c0] illegal short term buffer state detected [h264 @ 0x5589eb714b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb714b80] error while decoding MB 0 18, bytestream 8193 [h264 @ 0x5589eb7f1100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb7f1100] er", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 127, "h264_decode_error_count": 425, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:14:33+09:00", "last_error_summary": "[h264 @ 0x5589eb6c15c0] reference picture missing during reorder [h264 @ 0x5589eb6c15c0] Missing reference picture, default is 65562 [h264 @ 0x5589eb6d56c0] mmco: unref short failure [h264 @ 0x5589eb6d56c0] illegal short term buffer state detected [h264 @ 0x5589eb714b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb714b80] error while decoding MB 0 18, bytestream 8193 [h264 @ 0x5589eb7f1100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb7f1100] er", "last_exit_at": "2026-07-06T23:14:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 389.289, "last_started_at": "2026-07-06T23:08:03+09:00", "pid": 34549, "process": {"exists": false, "pid": 34549}, "restart_count": 136, "restart_count_per_hour": 7, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 137}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.412, "sample_count": 2510, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:14:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T23:14:33+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T23:14:33+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T23:14:33.009669+09:00", "mtime_epoch": 1783347273.009669, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124258}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:14:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1681.879, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.87, "pushed": 2.87, "read": 2.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.709, "count": 86, "max_ms": 5.416, "p95_ms": 5.012}, "gstreamer_push": {"avg_ms": 1.313, "count": 86, "max_ms": 1.538, "p95_ms": 1.432}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.208, "count": 86, "max_ms": 89.062, "p95_ms": 86.347}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.89, "count": 86, "max_ms": 115.964, "p95_ms": 107.039}, "yolo11_640_fire_11class": {"avg_ms": 76.809, "count": 86, "max_ms": 89.044, "p95_ms": 87.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.875, "count": 86, "max_ms": 101.193, "p95_ms": 95.696}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.668, "count": 86, "max_ms": 128.021, "p95_ms": 119.384}, "read_interval": {"avg_ms": 352.074, "count": 85, "max_ms": 20544.316, "p95_ms": 132.608}, "triton_wait": {"avg_ms": 86.226, "count": 86, "max_ms": 115.978, "p95_ms": 107.051}, "visualize": {"avg_ms": 3.644, "count": 86, "max_ms": 4.219, "p95_ms": 3.925}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6136.266, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 127, "h264_decode_error_count": 425, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:14:33+09:00", "last_error_summary": "[h264 @ 0x5589eb6c15c0] reference picture missing during reorder [h264 @ 0x5589eb6c15c0] Missing reference picture, default is 65562 [h264 @ 0x5589eb6d56c0] mmco: unref short failure [h264 @ 0x5589eb6d56c0] illegal short term buffer state detected [h264 @ 0x5589eb714b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb714b80] error while decoding MB 0 18, bytestream 8193 [h264 @ 0x5589eb7f1100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb7f1100] er", "last_exit_at": "2026-07-06T23:14:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 389.296, "last_started_at": "2026-07-06T23:08:03+09:00", "pid": 34549, "process": {"exists": false, "pid": 34549}, "restart_count": 136, "restart_count_per_hour": 7, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 137}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.412, "sample_count": 2510, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:14:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T23:14:33+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-06T23:14:33+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T23:14:33.009669+09:00", "mtime_epoch": 1783347273.009669, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124258}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:14:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1681.879, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.87, "pushed": 2.87, "read": 2.836}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.709, "count": 86, "max_ms": 5.416, "p95_ms": 5.012}, "gstreamer_push": {"avg_ms": 1.313, "count": 86, "max_ms": 1.538, "p95_ms": 1.432}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.208, "count": 86, "max_ms": 89.062, "p95_ms": 86.347}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.89, "count": 86, "max_ms": 115.964, "p95_ms": 107.039}, "yolo11_640_fire_11class": {"avg_ms": 76.809, "count": 86, "max_ms": 89.044, "p95_ms": 87.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.875, "count": 86, "max_ms": 101.193, "p95_ms": 95.696}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.668, "count": 86, "max_ms": 128.021, "p95_ms": 119.384}, "read_interval": {"avg_ms": 352.074, "count": 85, "max_ms": 20544.316, "p95_ms": 132.608}, "triton_wait": {"avg_ms": 86.226, "count": 86, "max_ms": 115.978, "p95_ms": 107.051}, "visualize": {"avg_ms": 3.644, "count": 86, "max_ms": 4.219, "p95_ms": 3.925}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6246.648, "limit_mb": 8796093022207.996}, "detail": {"pid": 34656, "restart_count": 137, "start_count": 138}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 127, "h264_decode_error_count": 425, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:14:33+09:00", "last_error_summary": "[h264 @ 0x5589eb6c15c0] reference picture missing during reorder [h264 @ 0x5589eb6c15c0] Missing reference picture, default is 65562 [h264 @ 0x5589eb6d56c0] mmco: unref short failure [h264 @ 0x5589eb6d56c0] illegal short term buffer state detected [h264 @ 0x5589eb714b80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb714b80] error while decoding MB 0 18, bytestream 8193 [h264 @ 0x5589eb7f1100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5589eb7f1100] er", "last_exit_at": "2026-07-06T23:14:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:14:35+09:00", "pid": 34656, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.41, "name": "ffmpeg", "pid": 34656, "rss_mb": 142.41, "state": "S (sleeping)", "threads": 106}, "restart_count": 137, "restart_count_per_hour": 8, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 138}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.506, "sample_count": 2466, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:14:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.715, "last_frame_pushed_at": "2026-07-06T23:14:33+09:00", "last_frame_read_age_sec": 4.82, "last_frame_read_at": "2026-07-06T23:14:33+09:00", "mjpeg": {"age_sec": 4.699, "exists": true, "mtime": "2026-07-06T23:14:33.125671+09:00", "mtime_epoch": 1783347273.1256707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121248}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:14:37+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1681.879, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.469, "pushed": 1.469, "read": 1.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.642, "count": 44, "max_ms": 5.416, "p95_ms": 4.909}, "gstreamer_push": {"avg_ms": 1.305, "count": 44, "max_ms": 1.444, "p95_ms": 1.432}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.916, "count": 44, "max_ms": 89.062, "p95_ms": 86.347}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.787, "count": 44, "max_ms": 115.964, "p95_ms": 107.586}, "yolo11_640_fire_11class": {"avg_ms": 76.91, "count": 44, "max_ms": 89.044, "p95_ms": 85.622}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.517, "count": 44, "max_ms": 101.193, "p95_ms": 92.956}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.506, "count": 44, "max_ms": 128.021, "p95_ms": 119.959}, "read_interval": {"avg_ms": 587.553, "count": 43, "max_ms": 20544.316, "p95_ms": 136.563}, "triton_wait": {"avg_ms": 87.255, "count": 44, "max_ms": 115.978, "p95_ms": 107.6}, "visualize": {"avg_ms": 3.612, "count": 44, "max_ms": 4.219, "p95_ms": 3.938}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6072.836, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56460c4e1800] error while decoding MB 91 37, bytestream -29 [in#0/rtsp @ 0x56460c49fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56460c49fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:14:54+09:00", "last_error_summary": "[h264 @ 0x56460c4e1800] error while decoding MB 91 37, bytestream -29 [in#0/rtsp @ 0x56460c49fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56460c49fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T22:51:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 946.68, "last_started_at": "2026-07-06T22:59:08+09:00", "pid": 34290, "process": {"exists": false, "pid": 34290}, "restart_count": 85, "restart_count_per_hour": 12, "rtsp_timeout_count": 134, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.947, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 319666, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:14:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-06T23:14:54+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T23:14:54+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T23:14:54.689981+09:00", "mtime_epoch": 1783347294.6899807, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:14:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1565.141, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.842, "pushed": 3.842, "read": 3.842}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.27, "count": 115, "max_ms": 6.313, "p95_ms": 5.189}, "gstreamer_push": {"avg_ms": 1.276, "count": 115, "max_ms": 1.449, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.395, "count": 115, "max_ms": 82.864, "p95_ms": 76.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.127, "count": 115, "max_ms": 112.718, "p95_ms": 86.616}, "yolo11_640_fire_11class": {"avg_ms": 60.667, "count": 115, "max_ms": 83.381, "p95_ms": 77.88}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.437, "count": 115, "max_ms": 91.127, "p95_ms": 88.708}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.587, "count": 115, "max_ms": 125.635, "p95_ms": 101.962}, "read_interval": {"avg_ms": 260.598, "count": 115, "max_ms": 10163.79, "p95_ms": 117.918}, "triton_wait": {"avg_ms": 67.685, "count": 115, "max_ms": 112.73, "p95_ms": 88.92}, "visualize": {"avg_ms": 3.657, "count": 115, "max_ms": 4.541, "p95_ms": 4.017}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6072.355, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:14:54+09:00", "last_error_summary": "[h264 @ 0x56460c4e1800] error while decoding MB 91 37, bytestream -29 [in#0/rtsp @ 0x56460c49fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56460c49fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:14:54+09:00", "last_error_summary": "[h264 @ 0x56460c4e1800] error while decoding MB 91 37, bytestream -29 [in#0/rtsp @ 0x56460c49fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56460c49fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:14:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 946.706, "last_started_at": "2026-07-06T22:59:08+09:00", "pid": 34290, "process": {"exists": false, "pid": 34290}, "restart_count": 85, "restart_count_per_hour": 12, "rtsp_timeout_count": 134, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.947, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 319667, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:14:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-06T23:14:54+09:00", "last_frame_read_age_sec": 0.087, "last_frame_read_at": "2026-07-06T23:14:54+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-06T23:14:54.689981+09:00", "mtime_epoch": 1783347294.6899807, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:14:54+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1565.141, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.872, "pushed": 3.872, "read": 3.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.322, "count": 116, "max_ms": 10.275, "p95_ms": 5.235}, "gstreamer_push": {"avg_ms": 1.278, "count": 116, "max_ms": 1.453, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.383, "count": 116, "max_ms": 82.864, "p95_ms": 76.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.041, "count": 116, "max_ms": 112.718, "p95_ms": 86.616}, "yolo11_640_fire_11class": {"avg_ms": 60.634, "count": 116, "max_ms": 83.381, "p95_ms": 77.88}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.394, "count": 116, "max_ms": 91.127, "p95_ms": 88.708}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.563, "count": 116, "max_ms": 125.635, "p95_ms": 101.962}, "read_interval": {"avg_ms": 260.598, "count": 115, "max_ms": 10163.79, "p95_ms": 117.918}, "triton_wait": {"avg_ms": 67.597, "count": 116, "max_ms": 112.73, "p95_ms": 88.92}, "visualize": {"avg_ms": 3.67, "count": 116, "max_ms": 5.097, "p95_ms": 4.054}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6072.336, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:14:54+09:00", "last_error_summary": "[h264 @ 0x56460c4e1800] error while decoding MB 91 37, bytestream -29 [in#0/rtsp @ 0x56460c49fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56460c49fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:14:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 946.715, "last_started_at": "2026-07-06T22:59:08+09:00", "pid": 34290, "process": {"exists": false, "pid": 34290}, "restart_count": 85, "restart_count_per_hour": 12, "rtsp_timeout_count": 134, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.947, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 319667, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:14:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T23:14:54+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-06T23:14:54+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-06T23:14:54.689981+09:00", "mtime_epoch": 1783347294.6899807, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:14:54+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1565.141, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.871, "pushed": 3.871, "read": 3.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.322, "count": 116, "max_ms": 10.275, "p95_ms": 5.235}, "gstreamer_push": {"avg_ms": 1.278, "count": 116, "max_ms": 1.453, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.383, "count": 116, "max_ms": 82.864, "p95_ms": 76.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.041, "count": 116, "max_ms": 112.718, "p95_ms": 86.616}, "yolo11_640_fire_11class": {"avg_ms": 60.634, "count": 116, "max_ms": 83.381, "p95_ms": 77.88}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.394, "count": 116, "max_ms": 91.127, "p95_ms": 88.708}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.563, "count": 116, "max_ms": 125.635, "p95_ms": 101.962}, "read_interval": {"avg_ms": 260.598, "count": 115, "max_ms": 10163.79, "p95_ms": 117.918}, "triton_wait": {"avg_ms": 67.597, "count": 116, "max_ms": 112.73, "p95_ms": 88.92}, "visualize": {"avg_ms": 3.67, "count": 116, "max_ms": 5.097, "p95_ms": 4.054}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6254.516, "limit_mb": 8796093022207.996}, "detail": {"pid": 34763, "restart_count": 86, "start_count": 87}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:14:54+09:00", "last_error_summary": "[h264 @ 0x56460c4e1800] error while decoding MB 91 37, bytestream -29 [in#0/rtsp @ 0x56460c49fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56460c49fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:14:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:14:57+09:00", "pid": 34763, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.684, "name": "ffmpeg", "pid": 34763, "rss_mb": 206.684, "state": "S (sleeping)", "threads": 106}, "restart_count": 86, "restart_count_per_hour": 13, "rtsp_timeout_count": 134, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.008, "sample_count": 2517, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 319667, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:14:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.812, "last_frame_pushed_at": "2026-07-06T23:14:54+09:00", "last_frame_read_age_sec": 4.892, "last_frame_read_at": "2026-07-06T23:14:54+09:00", "mjpeg": {"age_sec": 4.797, "exists": true, "mtime": "2026-07-06T23:14:54.781982+09:00", "mtime_epoch": 1783347294.781982, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:14:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1565.141, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.104, "pushed": 2.104, "read": 2.07}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.308, "count": 63, "max_ms": 10.275, "p95_ms": 5.057}, "gstreamer_push": {"avg_ms": 1.27, "count": 63, "max_ms": 1.453, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.533, "count": 63, "max_ms": 82.864, "p95_ms": 79.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.295, "count": 63, "max_ms": 112.718, "p95_ms": 88.373}, "yolo11_640_fire_11class": {"avg_ms": 61.874, "count": 63, "max_ms": 83.381, "p95_ms": 79.802}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.425, "count": 63, "max_ms": 91.127, "p95_ms": 89.646}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.563, "count": 63, "max_ms": 125.635, "p95_ms": 102.478}, "read_interval": {"avg_ms": 405.262, "count": 62, "max_ms": 10163.79, "p95_ms": 131.894}, "triton_wait": {"avg_ms": 69.636, "count": 63, "max_ms": 112.73, "p95_ms": 89.913}, "visualize": {"avg_ms": 3.663, "count": 63, "max_ms": 5.097, "p95_ms": 4.097}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2673}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6091.461, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b1404e1180] error while decoding MB 107 22, bytestream -5 [in#0/rtsp @ 0x55b140459e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b140459e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:17:36+09:00", "last_error_summary": "[h264 @ 0x55b1404e1180] error while decoding MB 107 22, bytestream -5 [in#0/rtsp @ 0x55b140459e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b140459e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:14:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.145, "last_started_at": "2026-07-06T23:14:57+09:00", "pid": 34763, "process": {"exists": false, "pid": 34763}, "restart_count": 86, "restart_count_per_hour": 12, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.287, "sample_count": 2258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 320820, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:17:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-06T23:17:36+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T23:17:36+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-06T23:17:36.660315+09:00", "mtime_epoch": 1783347456.6603146, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101313}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:17:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1565.656, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 19, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.635, "pushed": 0.635, "read": 0.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.304, "count": 19, "max_ms": 6.416, "p95_ms": 6.416}, "gstreamer_push": {"avg_ms": 1.322, "count": 19, "max_ms": 1.497, "p95_ms": 1.497}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.649, "count": 19, "max_ms": 79.559, "p95_ms": 79.559}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.577, "count": 19, "max_ms": 101.464, "p95_ms": 101.464}, "yolo11_640_fire_11class": {"avg_ms": 74.85, "count": 19, "max_ms": 87.755, "p95_ms": 87.755}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.195, "count": 19, "max_ms": 89.515, "p95_ms": 89.515}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.66, "count": 19, "max_ms": 114.357, "p95_ms": 114.357}, "read_interval": {"avg_ms": 1577.238, "count": 19, "max_ms": 15858.913, "p95_ms": 15858.913}, "triton_wait": {"avg_ms": 82.704, "count": 19, "max_ms": 101.476, "p95_ms": 101.476}, "visualize": {"avg_ms": 3.644, "count": 19, "max_ms": 4.603, "p95_ms": 4.603}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2673}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6090.836, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:17:36+09:00", "last_error_summary": "[h264 @ 0x55b1404e1180] error while decoding MB 107 22, bytestream -5 [in#0/rtsp @ 0x55b140459e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b140459e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:17:36+09:00", "last_error_summary": "[h264 @ 0x55b1404e1180] error while decoding MB 107 22, bytestream -5 [in#0/rtsp @ 0x55b140459e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b140459e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:17:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.187, "last_started_at": "2026-07-06T23:14:57+09:00", "pid": 34763, "process": {"exists": false, "pid": 34763}, "restart_count": 86, "restart_count_per_hour": 12, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.287, "sample_count": 2258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 320821, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:17:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T23:17:36+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-06T23:17:36+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-06T23:17:36.660315+09:00", "mtime_epoch": 1783347456.6603146, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101313}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:17:36+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1565.656, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 20, "pushed_frames": 20, "read_frames": 19, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.667, "pushed": 0.667, "read": 0.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.315, "count": 20, "max_ms": 6.416, "p95_ms": 5.604}, "gstreamer_push": {"avg_ms": 1.324, "count": 20, "max_ms": 1.497, "p95_ms": 1.429}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.81, "count": 20, "max_ms": 79.559, "p95_ms": 78.633}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.804, "count": 20, "max_ms": 101.464, "p95_ms": 96.724}, "yolo11_640_fire_11class": {"avg_ms": 75.012, "count": 20, "max_ms": 87.755, "p95_ms": 86.951}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.476, "count": 20, "max_ms": 89.515, "p95_ms": 88.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.784, "count": 20, "max_ms": 114.357, "p95_ms": 109.696}, "read_interval": {"avg_ms": 1577.238, "count": 19, "max_ms": 15858.913, "p95_ms": 15858.913}, "triton_wait": {"avg_ms": 82.81, "count": 20, "max_ms": 101.476, "p95_ms": 96.736}, "visualize": {"avg_ms": 3.649, "count": 20, "max_ms": 4.603, "p95_ms": 3.742}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2673}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6090.895, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:17:36+09:00", "last_error_summary": "[h264 @ 0x55b1404e1180] error while decoding MB 107 22, bytestream -5 [in#0/rtsp @ 0x55b140459e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b140459e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:17:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.194, "last_started_at": "2026-07-06T23:14:57+09:00", "pid": 34763, "process": {"exists": false, "pid": 34763}, "restart_count": 86, "restart_count_per_hour": 12, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.287, "sample_count": 2258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 320821, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:17:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T23:17:36+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-06T23:17:36+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T23:17:36.660315+09:00", "mtime_epoch": 1783347456.6603146, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101313}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:17:36+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1565.656, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 20, "pushed_frames": 20, "read_frames": 19, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.667, "pushed": 0.667, "read": 0.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.315, "count": 20, "max_ms": 6.416, "p95_ms": 5.604}, "gstreamer_push": {"avg_ms": 1.324, "count": 20, "max_ms": 1.497, "p95_ms": 1.429}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.81, "count": 20, "max_ms": 79.559, "p95_ms": 78.633}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.804, "count": 20, "max_ms": 101.464, "p95_ms": 96.724}, "yolo11_640_fire_11class": {"avg_ms": 75.012, "count": 20, "max_ms": 87.755, "p95_ms": 86.951}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.476, "count": 20, "max_ms": 89.515, "p95_ms": 88.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.784, "count": 20, "max_ms": 114.357, "p95_ms": 109.696}, "read_interval": {"avg_ms": 1577.238, "count": 19, "max_ms": 15858.913, "p95_ms": 15858.913}, "triton_wait": {"avg_ms": 82.81, "count": 20, "max_ms": 101.476, "p95_ms": 96.736}, "visualize": {"avg_ms": 3.649, "count": 20, "max_ms": 4.603, "p95_ms": 3.742}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2673}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6109.715, "limit_mb": 8796093022207.996}, "detail": {"pid": 34915, "restart_count": 87, "start_count": 88}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:17:36+09:00", "last_error_summary": "[h264 @ 0x55b1404e1180] error while decoding MB 107 22, bytestream -5 [in#0/rtsp @ 0x55b140459e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b140459e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:17:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:17:39+09:00", "pid": 34915, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.094, "name": "ffmpeg", "pid": 34915, "rss_mb": 57.094, "state": "S (sleeping)", "threads": 1}, "restart_count": 87, "restart_count_per_hour": 13, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.414, "sample_count": 2214, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 320821, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:17:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.985, "last_frame_pushed_at": "2026-07-06T23:17:36+09:00", "last_frame_read_age_sec": 5.086, "last_frame_read_at": "2026-07-06T23:17:36+09:00", "mjpeg": {"age_sec": 4.973, "exists": true, "mtime": "2026-07-06T23:17:36.772316+09:00", "mtime_epoch": 1783347456.7723162, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102108}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:17:41+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1565.656, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.587, "pushed": 0.587, "read": 0.587}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.435, "count": 13, "max_ms": 6.416, "p95_ms": 6.416}, "gstreamer_push": {"avg_ms": 1.345, "count": 13, "max_ms": 1.497, "p95_ms": 1.497}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.238, "count": 13, "max_ms": 77.859, "p95_ms": 77.859}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.15, "count": 13, "max_ms": 101.464, "p95_ms": 101.464}, "yolo11_640_fire_11class": {"avg_ms": 74.602, "count": 13, "max_ms": 87.755, "p95_ms": 87.755}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.09, "count": 13, "max_ms": 89.515, "p95_ms": 89.515}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.244, "count": 13, "max_ms": 114.357, "p95_ms": 114.357}, "read_interval": {"avg_ms": 2253.688, "count": 13, "max_ms": 15858.913, "p95_ms": 15858.913}, "triton_wait": {"avg_ms": 81.126, "count": 13, "max_ms": 101.476, "p95_ms": 101.476}, "visualize": {"avg_ms": 3.688, "count": 13, "max_ms": 4.603, "p95_ms": 4.603}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.13, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2683}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6100.992, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5612c3741140] error while decoding MB 46 35, bytestream -5 [in#0/rtsp @ 0x5612c2bf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5612c2bf8e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:20:23+09:00", "last_error_summary": "[h264 @ 0x5612c3741140] error while decoding MB 46 35, bytestream -5 [in#0/rtsp @ 0x5612c2bf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5612c2bf8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:17:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.058, "last_started_at": "2026-07-06T23:17:39+09:00", "pid": 34915, "process": {"exists": false, "pid": 34915}, "restart_count": 87, "restart_count_per_hour": 12, "rtsp_timeout_count": 138, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.095, "sample_count": 2095, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 321979, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:20:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-06T23:20:23+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-06T23:20:23+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T23:20:23.742732+09:00", "mtime_epoch": 1783347623.7427316, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104100}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:20:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1566.172, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.669, "pushed": 0.669, "read": 0.724}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.766, "count": 12, "max_ms": 4.196, "p95_ms": 4.196}, "gstreamer_push": {"avg_ms": 1.331, "count": 12, "max_ms": 1.483, "p95_ms": 1.483}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.203, "count": 12, "max_ms": 80.085, "p95_ms": 80.085}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.599, "count": 12, "max_ms": 86.658, "p95_ms": 86.658}, "yolo11_640_fire_11class": {"avg_ms": 69.997, "count": 12, "max_ms": 86.011, "p95_ms": 86.011}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.524, "count": 12, "max_ms": 97.155, "p95_ms": 97.155}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.412, "count": 12, "max_ms": 108.158, "p95_ms": 108.158}, "read_interval": {"avg_ms": 2609.143, "count": 13, "max_ms": 16784.823, "p95_ms": 16784.823}, "triton_wait": {"avg_ms": 77.089, "count": 12, "max_ms": 97.166, "p95_ms": 97.166}, "visualize": {"avg_ms": 3.438, "count": 12, "max_ms": 3.77, "p95_ms": 3.77}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.948, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2683}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6100.25, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:20:23+09:00", "last_error_summary": "[h264 @ 0x5612c3741140] error while decoding MB 46 35, bytestream -5 [in#0/rtsp @ 0x5612c2bf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5612c2bf8e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:20:23+09:00", "last_error_summary": "[h264 @ 0x5612c3741140] error while decoding MB 46 35, bytestream -5 [in#0/rtsp @ 0x5612c2bf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5612c2bf8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:20:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.11, "last_started_at": "2026-07-06T23:17:39+09:00", "pid": 34915, "process": {"exists": false, "pid": 34915}, "restart_count": 87, "restart_count_per_hour": 12, "rtsp_timeout_count": 138, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.095, "sample_count": 2096, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 321980, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:20:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T23:20:23+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-06T23:20:23+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T23:20:23.742732+09:00", "mtime_epoch": 1783347623.7427316, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104100}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:20:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1566.172, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.722, "pushed": 0.722, "read": 0.722}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.792, "count": 13, "max_ms": 4.196, "p95_ms": 4.196}, "gstreamer_push": {"avg_ms": 1.344, "count": 13, "max_ms": 1.505, "p95_ms": 1.505}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.348, "count": 13, "max_ms": 80.085, "p95_ms": 80.085}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.466, "count": 13, "max_ms": 86.658, "p95_ms": 86.658}, "yolo11_640_fire_11class": {"avg_ms": 71.002, "count": 13, "max_ms": 86.011, "p95_ms": 86.011}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.868, "count": 13, "max_ms": 97.155, "p95_ms": 97.155}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.614, "count": 13, "max_ms": 108.158, "p95_ms": 108.158}, "read_interval": {"avg_ms": 2609.143, "count": 13, "max_ms": 16784.823, "p95_ms": 16784.823}, "triton_wait": {"avg_ms": 78.236, "count": 13, "max_ms": 97.166, "p95_ms": 97.166}, "visualize": {"avg_ms": 3.46, "count": 13, "max_ms": 3.77, "p95_ms": 3.77}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.001, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2683}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6100.262, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:20:23+09:00", "last_error_summary": "[h264 @ 0x5612c3741140] error while decoding MB 46 35, bytestream -5 [in#0/rtsp @ 0x5612c2bf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5612c2bf8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:20:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.117, "last_started_at": "2026-07-06T23:17:39+09:00", "pid": 34915, "process": {"exists": false, "pid": 34915}, "restart_count": 87, "restart_count_per_hour": 12, "rtsp_timeout_count": 138, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.095, "sample_count": 2096, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 321980, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:20:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T23:20:23+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-06T23:20:23+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-06T23:20:23.742732+09:00", "mtime_epoch": 1783347623.7427316, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104100}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:20:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1566.172, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.722, "pushed": 0.722, "read": 0.722}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.792, "count": 13, "max_ms": 4.196, "p95_ms": 4.196}, "gstreamer_push": {"avg_ms": 1.344, "count": 13, "max_ms": 1.505, "p95_ms": 1.505}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.348, "count": 13, "max_ms": 80.085, "p95_ms": 80.085}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.466, "count": 13, "max_ms": 86.658, "p95_ms": 86.658}, "yolo11_640_fire_11class": {"avg_ms": 71.002, "count": 13, "max_ms": 86.011, "p95_ms": 86.011}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.868, "count": 13, "max_ms": 97.155, "p95_ms": 97.155}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.614, "count": 13, "max_ms": 108.158, "p95_ms": 108.158}, "read_interval": {"avg_ms": 2609.143, "count": 13, "max_ms": 16784.823, "p95_ms": 16784.823}, "triton_wait": {"avg_ms": 78.236, "count": 13, "max_ms": 97.166, "p95_ms": 97.166}, "visualize": {"avg_ms": 3.46, "count": 13, "max_ms": 3.77, "p95_ms": 3.77}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.008, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2683}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6280.02, "limit_mb": 8796093022207.996}, "detail": {"pid": 35067, "restart_count": 88, "start_count": 89}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:20:23+09:00", "last_error_summary": "[h264 @ 0x5612c3741140] error while decoding MB 46 35, bytestream -5 [in#0/rtsp @ 0x5612c2bf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5612c2bf8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:20:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:20:26+09:00", "pid": 35067, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.074, "name": "ffmpeg", "pid": 35067, "rss_mb": 209.074, "state": "S (sleeping)", "threads": 106}, "restart_count": 88, "restart_count_per_hour": 13, "rtsp_timeout_count": 138, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.095, "sample_count": 2054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 321980, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:20:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.655, "last_frame_pushed_at": "2026-07-06T23:20:23+09:00", "last_frame_read_age_sec": 4.763, "last_frame_read_at": "2026-07-06T23:20:23+09:00", "mjpeg": {"age_sec": 4.637, "exists": true, "mtime": "2026-07-06T23:20:23.866733+09:00", "mtime_epoch": 1783347623.8667333, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104458}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:20:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1566.172, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.574, "pushed": 0.574, "read": 0.574}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.792, "count": 13, "max_ms": 4.196, "p95_ms": 4.196}, "gstreamer_push": {"avg_ms": 1.344, "count": 13, "max_ms": 1.505, "p95_ms": 1.505}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.348, "count": 13, "max_ms": 80.085, "p95_ms": 80.085}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.466, "count": 13, "max_ms": 86.658, "p95_ms": 86.658}, "yolo11_640_fire_11class": {"avg_ms": 71.002, "count": 13, "max_ms": 86.011, "p95_ms": 86.011}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.868, "count": 13, "max_ms": 97.155, "p95_ms": 97.155}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.614, "count": 13, "max_ms": 108.158, "p95_ms": 108.158}, "read_interval": {"avg_ms": 2609.143, "count": 13, "max_ms": 16784.823, "p95_ms": 16784.823}, "triton_wait": {"avg_ms": 78.236, "count": 13, "max_ms": 97.166, "p95_ms": 97.166}, "visualize": {"avg_ms": 3.46, "count": 13, "max_ms": 3.77, "p95_ms": 3.77}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.65, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6139.379, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c2d77fe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77fe4c0] error while decoding MB 0 40, bytestream 3686 [h264 @ 0x55c2d770d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d770d440] error while decoding MB 0 11, bytestream 6273 [h264 @ 0x55c2d77372c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77372c0] error while decoding MB 0 12, bytestream 1760 [h264 @ 0x55c2d8210fc0] error while decoding MB 15 25, b", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 128, "h264_decode_error_count": 429, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:22:01+09:00", "last_error_summary": "[h264 @ 0x55c2d77fe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77fe4c0] error while decoding MB 0 40, bytestream 3686 [h264 @ 0x55c2d770d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d770d440] error while decoding MB 0 11, bytestream 6273 [h264 @ 0x55c2d77372c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77372c0] error while decoding MB 0 12, bytestream 1760 [h264 @ 0x55c2d8210fc0] error while decoding MB 15 25, b", "last_exit_at": "2026-07-06T23:14:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 445.646, "last_started_at": "2026-07-06T23:14:35+09:00", "pid": 34656, "process": {"exists": false, "pid": 34656}, "restart_count": 137, "restart_count_per_hour": 8, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 138}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.686, "sample_count": 2660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 303329, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:22:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-06T23:22:01+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-06T23:22:01+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-06T23:22:01.412148+09:00", "mtime_epoch": 1783347721.4121478, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125566}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:22:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1683.684, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.3, "pushed": 3.3, "read": 3.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.548, "count": 99, "max_ms": 4.962, "p95_ms": 4.611}, "gstreamer_push": {"avg_ms": 1.314, "count": 99, "max_ms": 1.598, "p95_ms": 1.462}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.691, "count": 99, "max_ms": 88.893, "p95_ms": 86.156}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.746, "count": 99, "max_ms": 106.141, "p95_ms": 90.004}, "yolo11_640_fire_11class": {"avg_ms": 76.33, "count": 99, "max_ms": 89.493, "p95_ms": 87.99}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.102, "count": 99, "max_ms": 100.397, "p95_ms": 98.551}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.678, "count": 99, "max_ms": 118.367, "p95_ms": 112.552}, "read_interval": {"avg_ms": 300.45, "count": 100, "max_ms": 19206.542, "p95_ms": 125.395}, "triton_wait": {"avg_ms": 84.542, "count": 99, "max_ms": 106.153, "p95_ms": 100.406}, "visualize": {"avg_ms": 3.596, "count": 99, "max_ms": 4.223, "p95_ms": 4.023}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6138.609, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:22:01+09:00", "last_error_summary": "[h264 @ 0x55c2d77fe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77fe4c0] error while decoding MB 0 40, bytestream 3686 [h264 @ 0x55c2d770d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d770d440] error while decoding MB 0 11, bytestream 6273 [h264 @ 0x55c2d77372c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77372c0] error while decoding MB 0 12, bytestream 1760 [h264 @ 0x55c2d8210fc0] error while decoding MB 15 25, b", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 128, "h264_decode_error_count": 429, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:22:01+09:00", "last_error_summary": "[h264 @ 0x55c2d77fe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77fe4c0] error while decoding MB 0 40, bytestream 3686 [h264 @ 0x55c2d770d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d770d440] error while decoding MB 0 11, bytestream 6273 [h264 @ 0x55c2d77372c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77372c0] error while decoding MB 0 12, bytestream 1760 [h264 @ 0x55c2d8210fc0] error while decoding MB 15 25, b", "last_exit_at": "2026-07-06T23:22:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 445.695, "last_started_at": "2026-07-06T23:14:35+09:00", "pid": 34656, "process": {"exists": false, "pid": 34656}, "restart_count": 137, "restart_count_per_hour": 8, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 138}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.686, "sample_count": 2661, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 303330, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:22:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T23:22:01+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-06T23:22:01+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T23:22:01.412148+09:00", "mtime_epoch": 1783347721.4121478, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125566}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:22:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1683.684, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.338, "pushed": 3.338, "read": 3.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.546, "count": 100, "max_ms": 4.962, "p95_ms": 4.509}, "gstreamer_push": {"avg_ms": 1.313, "count": 100, "max_ms": 1.598, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.813, "count": 100, "max_ms": 88.893, "p95_ms": 85.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.81, "count": 100, "max_ms": 106.141, "p95_ms": 89.84}, "yolo11_640_fire_11class": {"avg_ms": 76.38, "count": 100, "max_ms": 89.493, "p95_ms": 87.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.169, "count": 100, "max_ms": 100.397, "p95_ms": 97.38}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.738, "count": 100, "max_ms": 118.367, "p95_ms": 112.367}, "read_interval": {"avg_ms": 302.355, "count": 99, "max_ms": 19206.542, "p95_ms": 125.551}, "triton_wait": {"avg_ms": 84.604, "count": 100, "max_ms": 106.153, "p95_ms": 99.523}, "visualize": {"avg_ms": 3.596, "count": 100, "max_ms": 4.223, "p95_ms": 3.943}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6138.598, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 128, "h264_decode_error_count": 429, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:22:01+09:00", "last_error_summary": "[h264 @ 0x55c2d77fe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77fe4c0] error while decoding MB 0 40, bytestream 3686 [h264 @ 0x55c2d770d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d770d440] error while decoding MB 0 11, bytestream 6273 [h264 @ 0x55c2d77372c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77372c0] error while decoding MB 0 12, bytestream 1760 [h264 @ 0x55c2d8210fc0] error while decoding MB 15 25, b", "last_exit_at": "2026-07-06T23:22:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 445.702, "last_started_at": "2026-07-06T23:14:35+09:00", "pid": 34656, "process": {"exists": false, "pid": 34656}, "restart_count": 137, "restart_count_per_hour": 8, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 138}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.686, "sample_count": 2660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 303330, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:22:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T23:22:01+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-06T23:22:01+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-06T23:22:01.412148+09:00", "mtime_epoch": 1783347721.4121478, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125566}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:22:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1683.684, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.337, "pushed": 3.337, "read": 3.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.546, "count": 100, "max_ms": 4.962, "p95_ms": 4.509}, "gstreamer_push": {"avg_ms": 1.313, "count": 100, "max_ms": 1.598, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.813, "count": 100, "max_ms": 88.893, "p95_ms": 85.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.81, "count": 100, "max_ms": 106.141, "p95_ms": 89.84}, "yolo11_640_fire_11class": {"avg_ms": 76.38, "count": 100, "max_ms": 89.493, "p95_ms": 87.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.169, "count": 100, "max_ms": 100.397, "p95_ms": 97.38}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.738, "count": 100, "max_ms": 118.367, "p95_ms": 112.367}, "read_interval": {"avg_ms": 302.355, "count": 99, "max_ms": 19206.542, "p95_ms": 125.551}, "triton_wait": {"avg_ms": 84.604, "count": 100, "max_ms": 106.153, "p95_ms": 99.523}, "visualize": {"avg_ms": 3.596, "count": 100, "max_ms": 4.223, "p95_ms": 3.943}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2527}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6246.414, "limit_mb": 8796093022207.996}, "detail": {"pid": 35219, "restart_count": 138, "start_count": 139}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 128, "h264_decode_error_count": 429, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:22:01+09:00", "last_error_summary": "[h264 @ 0x55c2d77fe4c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77fe4c0] error while decoding MB 0 40, bytestream 3686 [h264 @ 0x55c2d770d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d770d440] error while decoding MB 0 11, bytestream 6273 [h264 @ 0x55c2d77372c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c2d77372c0] error while decoding MB 0 12, bytestream 1760 [h264 @ 0x55c2d8210fc0] error while decoding MB 15 25, b", "last_exit_at": "2026-07-06T23:22:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:22:04+09:00", "pid": 35219, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.973, "name": "ffmpeg", "pid": 35219, "rss_mb": 138.973, "state": "S (sleeping)", "threads": 106}, "restart_count": 138, "restart_count_per_hour": 9, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 139}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.696, "sample_count": 2618, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 303330, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:22:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.629, "last_frame_pushed_at": "2026-07-06T23:22:01+09:00", "last_frame_read_age_sec": 4.733, "last_frame_read_at": "2026-07-06T23:22:01+09:00", "mjpeg": {"age_sec": 4.613, "exists": true, "mtime": "2026-07-06T23:22:01.528149+09:00", "mtime_epoch": 1783347721.5281494, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126888}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:22:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1683.684, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.934, "pushed": 1.934, "read": 1.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.581, "count": 58, "max_ms": 4.962, "p95_ms": 4.356}, "gstreamer_push": {"avg_ms": 1.33, "count": 58, "max_ms": 1.598, "p95_ms": 1.482}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.994, "count": 58, "max_ms": 88.893, "p95_ms": 88.628}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.994, "count": 58, "max_ms": 106.141, "p95_ms": 93.936}, "yolo11_640_fire_11class": {"avg_ms": 76.545, "count": 58, "max_ms": 89.493, "p95_ms": 88.12}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.235, "count": 58, "max_ms": 100.397, "p95_ms": 99.297}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.091, "count": 58, "max_ms": 118.367, "p95_ms": 112.696}, "read_interval": {"avg_ms": 444.761, "count": 57, "max_ms": 19206.542, "p95_ms": 127.584}, "triton_wait": {"avg_ms": 84.84, "count": 58, "max_ms": 106.153, "p95_ms": 100.411}, "visualize": {"avg_ms": 3.64, "count": 58, "max_ms": 4.093, "p95_ms": 3.943}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2705}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6105.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5582b63b5400] error while decoding MB 46 21, bytestream -7 [in#0/rtsp @ 0x5582b5ba7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582b5ba7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:25:55+09:00", "last_error_summary": "[h264 @ 0x5582b63b5400] error while decoding MB 46 21, bytestream -7 [in#0/rtsp @ 0x5582b5ba7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582b5ba7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:20:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 329.48, "last_started_at": "2026-07-06T23:20:26+09:00", "pid": 35067, "process": {"exists": false, "pid": 35067}, "restart_count": 88, "restart_count_per_hour": 12, "rtsp_timeout_count": 140, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.62, "sample_count": 2453, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 324677, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:25:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-06T23:25:55+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-06T23:25:55+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T23:25:55.923555+09:00", "mtime_epoch": 1783347955.9235551, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115724}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:25:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1567.461, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.676, "pushed": 0.676, "read": 0.733}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.09, "count": 12, "max_ms": 4.275, "p95_ms": 4.275}, "gstreamer_push": {"avg_ms": 1.3, "count": 12, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.872, "count": 12, "max_ms": 85.618, "p95_ms": 85.618}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.785, "count": 12, "max_ms": 100.116, "p95_ms": 100.116}, "yolo11_640_fire_11class": {"avg_ms": 75.197, "count": 12, "max_ms": 85.426, "p95_ms": 85.426}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.103, "count": 12, "max_ms": 100.875, "p95_ms": 100.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.712, "count": 12, "max_ms": 113.937, "p95_ms": 113.937}, "read_interval": {"avg_ms": 2605.998, "count": 13, "max_ms": 16439.267, "p95_ms": 16439.267}, "triton_wait": {"avg_ms": 84.859, "count": 12, "max_ms": 100.888, "p95_ms": 100.888}, "visualize": {"avg_ms": 3.758, "count": 12, "max_ms": 3.976, "p95_ms": 3.976}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.74, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2705}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6104.23, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:25:55+09:00", "last_error_summary": "[h264 @ 0x5582b63b5400] error while decoding MB 46 21, bytestream -7 [in#0/rtsp @ 0x5582b5ba7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582b5ba7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:25:55+09:00", "last_error_summary": "[h264 @ 0x5582b63b5400] error while decoding MB 46 21, bytestream -7 [in#0/rtsp @ 0x5582b5ba7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582b5ba7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:25:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 329.515, "last_started_at": "2026-07-06T23:20:26+09:00", "pid": 35067, "process": {"exists": false, "pid": 35067}, "restart_count": 88, "restart_count_per_hour": 12, "rtsp_timeout_count": 140, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.62, "sample_count": 2454, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 324678, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:25:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T23:25:56+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-06T23:25:55+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-06T23:25:55.923555+09:00", "mtime_epoch": 1783347955.9235551, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115724}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:25:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1567.461, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.075, "count": 13, "max_ms": 4.275, "p95_ms": 4.275}, "gstreamer_push": {"avg_ms": 1.29, "count": 13, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.14, "count": 13, "max_ms": 85.618, "p95_ms": 85.618}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.749, "count": 13, "max_ms": 100.116, "p95_ms": 100.116}, "yolo11_640_fire_11class": {"avg_ms": 75.134, "count": 13, "max_ms": 85.426, "p95_ms": 85.426}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.681, "count": 13, "max_ms": 100.875, "p95_ms": 100.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.201, "count": 13, "max_ms": 113.937, "p95_ms": 113.937}, "read_interval": {"avg_ms": 2605.998, "count": 13, "max_ms": 16439.267, "p95_ms": 16439.267}, "triton_wait": {"avg_ms": 84.379, "count": 13, "max_ms": 100.888, "p95_ms": 100.888}, "visualize": {"avg_ms": 3.74, "count": 13, "max_ms": 3.976, "p95_ms": 3.976}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.775, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2705}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6104.164, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:25:55+09:00", "last_error_summary": "[h264 @ 0x5582b63b5400] error while decoding MB 46 21, bytestream -7 [in#0/rtsp @ 0x5582b5ba7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582b5ba7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:25:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 329.522, "last_started_at": "2026-07-06T23:20:26+09:00", "pid": 35067, "process": {"exists": false, "pid": 35067}, "restart_count": 88, "restart_count_per_hour": 12, "rtsp_timeout_count": 140, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.62, "sample_count": 2454, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 324678, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:25:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T23:25:56+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-06T23:25:55+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T23:25:55.923555+09:00", "mtime_epoch": 1783347955.9235551, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115724}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:25:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1567.461, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.075, "count": 13, "max_ms": 4.275, "p95_ms": 4.275}, "gstreamer_push": {"avg_ms": 1.29, "count": 13, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.14, "count": 13, "max_ms": 85.618, "p95_ms": 85.618}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.749, "count": 13, "max_ms": 100.116, "p95_ms": 100.116}, "yolo11_640_fire_11class": {"avg_ms": 75.134, "count": 13, "max_ms": 85.426, "p95_ms": 85.426}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.681, "count": 13, "max_ms": 100.875, "p95_ms": 100.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.201, "count": 13, "max_ms": 113.937, "p95_ms": 113.937}, "read_interval": {"avg_ms": 2605.998, "count": 13, "max_ms": 16439.267, "p95_ms": 16439.267}, "triton_wait": {"avg_ms": 84.379, "count": 13, "max_ms": 100.888, "p95_ms": 100.888}, "visualize": {"avg_ms": 3.74, "count": 13, "max_ms": 3.976, "p95_ms": 3.976}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.782, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2705}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6122.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 35326, "restart_count": 89, "start_count": 90}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:25:55+09:00", "last_error_summary": "[h264 @ 0x5582b63b5400] error while decoding MB 46 21, bytestream -7 [in#0/rtsp @ 0x5582b5ba7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582b5ba7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:25:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:25:58+09:00", "pid": 35326, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.27, "name": "ffmpeg", "pid": 35326, "rss_mb": 58.27, "state": "R (running)", "threads": 1}, "restart_count": 89, "restart_count_per_hour": 13, "rtsp_timeout_count": 140, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.706, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 324678, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:25:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.762, "last_frame_pushed_at": "2026-07-06T23:25:56+09:00", "last_frame_read_age_sec": 4.855, "last_frame_read_at": "2026-07-06T23:25:55+09:00", "mjpeg": {"age_sec": 4.74, "exists": true, "mtime": "2026-07-06T23:25:56.035557+09:00", "mtime_epoch": 1783347956.0355568, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115982}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:26:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1567.461, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.577, "pushed": 0.577, "read": 0.577}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.075, "count": 13, "max_ms": 4.275, "p95_ms": 4.275}, "gstreamer_push": {"avg_ms": 1.29, "count": 13, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.14, "count": 13, "max_ms": 85.618, "p95_ms": 85.618}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.749, "count": 13, "max_ms": 100.116, "p95_ms": 100.116}, "yolo11_640_fire_11class": {"avg_ms": 75.134, "count": 13, "max_ms": 85.426, "p95_ms": 85.426}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.681, "count": 13, "max_ms": 100.875, "p95_ms": 100.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.201, "count": 13, "max_ms": 113.937, "p95_ms": 113.937}, "read_interval": {"avg_ms": 2605.998, "count": 13, "max_ms": 16439.267, "p95_ms": 16439.267}, "triton_wait": {"avg_ms": 84.379, "count": 13, "max_ms": 100.888, "p95_ms": 100.888}, "visualize": {"avg_ms": 3.74, "count": 13, "max_ms": 3.976, "p95_ms": 3.976}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.533, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2546}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6136.918, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ac62e9f780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac62e9f780] error while decoding MB 0 8, bytestream 3290 [h264 @ 0x55ac6230ad80] left block unavailable for requested intra mode [h264 @ 0x55ac6230ad80] error while decoding MB 0 38, bytestream 4001 [in#0/rtsp @ 0x55ac62286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac62286e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 129, "h264_decode_error_count": 431, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:26:22+09:00", "last_error_summary": "[h264 @ 0x55ac62e9f780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac62e9f780] error while decoding MB 0 8, bytestream 3290 [h264 @ 0x55ac6230ad80] left block unavailable for requested intra mode [h264 @ 0x55ac6230ad80] error while decoding MB 0 38, bytestream 4001 [in#0/rtsp @ 0x55ac62286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac62286e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:22:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 257.914, "last_started_at": "2026-07-06T23:22:04+09:00", "pid": 35219, "process": {"exists": false, "pid": 35219}, "restart_count": 138, "restart_count_per_hour": 8, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 139}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.394, "sample_count": 2403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 305548, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:26:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-06T23:26:21+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-06T23:26:21+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-06T23:26:22.003935+09:00", "mtime_epoch": 1783347982.0039346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122701}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:26:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1684.715, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.838, "pushed": 3.838, "read": 3.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.51, "count": 115, "max_ms": 4.42, "p95_ms": 3.967}, "gstreamer_push": {"avg_ms": 1.316, "count": 115, "max_ms": 1.515, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.465, "count": 115, "max_ms": 87.586, "p95_ms": 78.297}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.671, "count": 115, "max_ms": 109.658, "p95_ms": 84.334}, "yolo11_640_fire_11class": {"avg_ms": 61.929, "count": 115, "max_ms": 91.231, "p95_ms": 79.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.201, "count": 115, "max_ms": 101.06, "p95_ms": 89.296}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.543, "count": 115, "max_ms": 121.628, "p95_ms": 103.497}, "read_interval": {"avg_ms": 260.877, "count": 115, "max_ms": 19352.726, "p95_ms": 121.428}, "triton_wait": {"avg_ms": 68.457, "count": 115, "max_ms": 109.67, "p95_ms": 91.451}, "visualize": {"avg_ms": 3.647, "count": 115, "max_ms": 4.278, "p95_ms": 3.948}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2546}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6136.383, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:26:22+09:00", "last_error_summary": "[h264 @ 0x55ac62e9f780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac62e9f780] error while decoding MB 0 8, bytestream 3290 [h264 @ 0x55ac6230ad80] left block unavailable for requested intra mode [h264 @ 0x55ac6230ad80] error while decoding MB 0 38, bytestream 4001 [in#0/rtsp @ 0x55ac62286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac62286e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 129, "h264_decode_error_count": 431, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:26:22+09:00", "last_error_summary": "[h264 @ 0x55ac62e9f780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac62e9f780] error while decoding MB 0 8, bytestream 3290 [h264 @ 0x55ac6230ad80] left block unavailable for requested intra mode [h264 @ 0x55ac6230ad80] error while decoding MB 0 38, bytestream 4001 [in#0/rtsp @ 0x55ac62286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac62286e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:26:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 257.958, "last_started_at": "2026-07-06T23:22:04+09:00", "pid": 35219, "process": {"exists": false, "pid": 35219}, "restart_count": 138, "restart_count_per_hour": 8, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 139}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.394, "sample_count": 2403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 305549, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:26:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T23:26:22+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-06T23:26:21+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-06T23:26:22.003935+09:00", "mtime_epoch": 1783347982.0039346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122701}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:26:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1684.715, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.834, "pushed": 3.834, "read": 3.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.51, "count": 115, "max_ms": 4.42, "p95_ms": 3.967}, "gstreamer_push": {"avg_ms": 1.316, "count": 115, "max_ms": 1.515, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.538, "count": 115, "max_ms": 87.586, "p95_ms": 78.297}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.59, "count": 115, "max_ms": 109.658, "p95_ms": 84.334}, "yolo11_640_fire_11class": {"avg_ms": 62.031, "count": 115, "max_ms": 91.231, "p95_ms": 79.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.358, "count": 115, "max_ms": 101.06, "p95_ms": 89.296}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.538, "count": 115, "max_ms": 121.628, "p95_ms": 103.497}, "read_interval": {"avg_ms": 260.877, "count": 115, "max_ms": 19352.726, "p95_ms": 121.428}, "triton_wait": {"avg_ms": 68.454, "count": 115, "max_ms": 109.67, "p95_ms": 91.451}, "visualize": {"avg_ms": 3.644, "count": 115, "max_ms": 4.278, "p95_ms": 3.93}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2546}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6136.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 129, "h264_decode_error_count": 431, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:26:22+09:00", "last_error_summary": "[h264 @ 0x55ac62e9f780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac62e9f780] error while decoding MB 0 8, bytestream 3290 [h264 @ 0x55ac6230ad80] left block unavailable for requested intra mode [h264 @ 0x55ac6230ad80] error while decoding MB 0 38, bytestream 4001 [in#0/rtsp @ 0x55ac62286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac62286e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:26:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 257.967, "last_started_at": "2026-07-06T23:22:04+09:00", "pid": 35219, "process": {"exists": false, "pid": 35219}, "restart_count": 138, "restart_count_per_hour": 8, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 139}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.394, "sample_count": 2403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 305549, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:26:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T23:26:22+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T23:26:21+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-06T23:26:22.003935+09:00", "mtime_epoch": 1783347982.0039346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122701}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:26:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1684.715, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.842, "pushed": 3.842, "read": 3.808}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.51, "count": 115, "max_ms": 4.42, "p95_ms": 3.967}, "gstreamer_push": {"avg_ms": 1.316, "count": 115, "max_ms": 1.515, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.538, "count": 115, "max_ms": 87.586, "p95_ms": 78.297}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.59, "count": 115, "max_ms": 109.658, "p95_ms": 84.334}, "yolo11_640_fire_11class": {"avg_ms": 62.031, "count": 115, "max_ms": 91.231, "p95_ms": 79.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.358, "count": 115, "max_ms": 101.06, "p95_ms": 89.296}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.538, "count": 115, "max_ms": 121.628, "p95_ms": 103.497}, "read_interval": {"avg_ms": 262.244, "count": 114, "max_ms": 19352.726, "p95_ms": 121.428}, "triton_wait": {"avg_ms": 68.454, "count": 115, "max_ms": 109.67, "p95_ms": 91.451}, "visualize": {"avg_ms": 3.644, "count": 115, "max_ms": 4.278, "p95_ms": 3.93}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2546}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6244.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 35478, "restart_count": 139, "start_count": 140}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 129, "h264_decode_error_count": 431, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:26:22+09:00", "last_error_summary": "[h264 @ 0x55ac62e9f780] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac62e9f780] error while decoding MB 0 8, bytestream 3290 [h264 @ 0x55ac6230ad80] left block unavailable for requested intra mode [h264 @ 0x55ac6230ad80] error while decoding MB 0 38, bytestream 4001 [in#0/rtsp @ 0x55ac62286e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac62286e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:26:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:26:24+09:00", "pid": 35478, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.969, "name": "ffmpeg", "pid": 35478, "rss_mb": 138.969, "state": "S (sleeping)", "threads": 106}, "restart_count": 139, "restart_count_per_hour": 9, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.394, "sample_count": 2362, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 305549, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:26:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.621, "last_frame_pushed_at": "2026-07-06T23:26:22+09:00", "last_frame_read_age_sec": 4.715, "last_frame_read_at": "2026-07-06T23:26:21+09:00", "mjpeg": {"age_sec": 4.605, "exists": true, "mtime": "2026-07-06T23:26:22.107936+09:00", "mtime_epoch": 1783347982.1079361, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121595}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:26:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1684.715, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 65, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.205, "pushed": 2.205, "read": 2.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.499, "count": 66, "max_ms": 4.087, "p95_ms": 3.851}, "gstreamer_push": {"avg_ms": 1.314, "count": 66, "max_ms": 1.515, "p95_ms": 1.441}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.565, "count": 66, "max_ms": 87.586, "p95_ms": 78.297}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.704, "count": 66, "max_ms": 109.658, "p95_ms": 84.334}, "yolo11_640_fire_11class": {"avg_ms": 61.951, "count": 66, "max_ms": 91.231, "p95_ms": 81.525}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.154, "count": 66, "max_ms": 101.06, "p95_ms": 88.887}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.537, "count": 66, "max_ms": 121.628, "p95_ms": 100.862}, "read_interval": {"avg_ms": 389.097, "count": 65, "max_ms": 19352.726, "p95_ms": 113.459}, "triton_wait": {"avg_ms": 68.462, "count": 66, "max_ms": 109.67, "p95_ms": 88.9}, "visualize": {"avg_ms": 3.637, "count": 66, "max_ms": 4.278, "p95_ms": 4.042}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2737}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6092.719, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556315ac5c40] error while decoding MB 32 34, bytestream -7 [in#0/rtsp @ 0x556315044e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556315044e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 90, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:33:26+09:00", "last_error_summary": "[h264 @ 0x556315ac5c40] error while decoding MB 32 34, bytestream -7 [in#0/rtsp @ 0x556315044e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556315044e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:25:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 448.095, "last_started_at": "2026-07-06T23:25:58+09:00", "pid": 35326, "process": {"exists": false, "pid": 35326}, "restart_count": 89, "restart_count_per_hour": 11, "rtsp_timeout_count": 142, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.835, "sample_count": 2505, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 328516, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:33:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-06T23:33:26+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-06T23:33:26+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T23:33:26.810124+09:00", "mtime_epoch": 1783348406.8101244, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 120166}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:33:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1569.266, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.268, "pushed": 3.268, "read": 3.268}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.412, "count": 98, "max_ms": 7.0, "p95_ms": 6.66}, "gstreamer_push": {"avg_ms": 1.299, "count": 98, "max_ms": 1.845, "p95_ms": 1.537}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.075, "count": 98, "max_ms": 87.899, "p95_ms": 84.191}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.387, "count": 98, "max_ms": 109.521, "p95_ms": 106.611}, "yolo11_640_fire_11class": {"avg_ms": 76.025, "count": 98, "max_ms": 90.174, "p95_ms": 85.358}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.545, "count": 98, "max_ms": 99.608, "p95_ms": 95.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.745, "count": 98, "max_ms": 122.35, "p95_ms": 119.735}, "read_interval": {"avg_ms": 306.343, "count": 98, "max_ms": 10160.558, "p95_ms": 133.224}, "triton_wait": {"avg_ms": 86.505, "count": 98, "max_ms": 109.534, "p95_ms": 106.623}, "visualize": {"avg_ms": 3.726, "count": 98, "max_ms": 6.06, "p95_ms": 4.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2737}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6091.773, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:33:26+09:00", "last_error_summary": "[h264 @ 0x556315ac5c40] error while decoding MB 32 34, bytestream -7 [in#0/rtsp @ 0x556315044e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556315044e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 90, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:33:26+09:00", "last_error_summary": "[h264 @ 0x556315ac5c40] error while decoding MB 32 34, bytestream -7 [in#0/rtsp @ 0x556315044e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556315044e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:33:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 448.13, "last_started_at": "2026-07-06T23:25:58+09:00", "pid": 35326, "process": {"exists": false, "pid": 35326}, "restart_count": 89, "restart_count_per_hour": 11, "rtsp_timeout_count": 142, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.835, "sample_count": 2505, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 328517, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:33:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T23:33:26+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-06T23:33:26+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-06T23:33:26.810124+09:00", "mtime_epoch": 1783348406.8101244, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 120166}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:33:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1569.266, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.276, "pushed": 3.276, "read": 3.243}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.411, "count": 98, "max_ms": 7.0, "p95_ms": 6.66}, "gstreamer_push": {"avg_ms": 1.298, "count": 98, "max_ms": 1.845, "p95_ms": 1.537}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.059, "count": 98, "max_ms": 87.899, "p95_ms": 84.191}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.398, "count": 98, "max_ms": 109.521, "p95_ms": 106.611}, "yolo11_640_fire_11class": {"avg_ms": 76.015, "count": 98, "max_ms": 90.174, "p95_ms": 85.358}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.484, "count": 98, "max_ms": 99.608, "p95_ms": 95.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.682, "count": 98, "max_ms": 122.35, "p95_ms": 119.735}, "read_interval": {"avg_ms": 308.338, "count": 97, "max_ms": 10160.558, "p95_ms": 133.224}, "triton_wait": {"avg_ms": 86.445, "count": 98, "max_ms": 109.534, "p95_ms": 106.623}, "visualize": {"avg_ms": 3.727, "count": 98, "max_ms": 6.06, "p95_ms": 4.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2737}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6091.613, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 90, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:33:26+09:00", "last_error_summary": "[h264 @ 0x556315ac5c40] error while decoding MB 32 34, bytestream -7 [in#0/rtsp @ 0x556315044e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556315044e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:33:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 448.137, "last_started_at": "2026-07-06T23:25:58+09:00", "pid": 35326, "process": {"exists": false, "pid": 35326}, "restart_count": 89, "restart_count_per_hour": 11, "rtsp_timeout_count": 142, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.835, "sample_count": 2505, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 328517, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:33:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T23:33:26+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T23:33:26+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-06T23:33:26.810124+09:00", "mtime_epoch": 1783348406.8101244, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 120166}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:33:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1569.266, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.276, "pushed": 3.276, "read": 3.242}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.411, "count": 98, "max_ms": 7.0, "p95_ms": 6.66}, "gstreamer_push": {"avg_ms": 1.298, "count": 98, "max_ms": 1.845, "p95_ms": 1.537}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.059, "count": 98, "max_ms": 87.899, "p95_ms": 84.191}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.398, "count": 98, "max_ms": 109.521, "p95_ms": 106.611}, "yolo11_640_fire_11class": {"avg_ms": 76.015, "count": 98, "max_ms": 90.174, "p95_ms": 85.358}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.484, "count": 98, "max_ms": 99.608, "p95_ms": 95.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.682, "count": 98, "max_ms": 122.35, "p95_ms": 119.735}, "read_interval": {"avg_ms": 308.338, "count": 97, "max_ms": 10160.558, "p95_ms": 133.224}, "triton_wait": {"avg_ms": 86.445, "count": 98, "max_ms": 109.534, "p95_ms": 106.623}, "visualize": {"avg_ms": 3.727, "count": 98, "max_ms": 6.06, "p95_ms": 4.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2737}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6157.77, "limit_mb": 8796093022207.996}, "detail": {"pid": 35585, "restart_count": 90, "start_count": 91}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 90, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:33:26+09:00", "last_error_summary": "[h264 @ 0x556315ac5c40] error while decoding MB 32 34, bytestream -7 [in#0/rtsp @ 0x556315044e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556315044e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:33:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:33:28+09:00", "pid": 35585, "process": {"exists": true, "fd_count": 5, "hwm_mb": 90.609, "name": "ffmpeg", "pid": 35585, "rss_mb": 90.609, "state": "S (sleeping)", "threads": 19}, "restart_count": 90, "restart_count_per_hour": 12, "rtsp_timeout_count": 142, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.835, "sample_count": 2469, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 328517, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:33:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.081, "last_frame_pushed_at": "2026-07-06T23:33:26+09:00", "last_frame_read_age_sec": 4.176, "last_frame_read_at": "2026-07-06T23:33:26+09:00", "mjpeg": {"age_sec": 4.065, "exists": true, "mtime": "2026-07-06T23:33:26.914126+09:00", "mtime_epoch": 1783348406.914126, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 120948}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:33:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1569.266, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 62, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.067, "pushed": 2.067, "read": 2.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.501, "count": 62, "max_ms": 7.0, "p95_ms": 6.815}, "gstreamer_push": {"avg_ms": 1.316, "count": 62, "max_ms": 1.845, "p95_ms": 1.686}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.929, "count": 62, "max_ms": 87.899, "p95_ms": 82.042}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.906, "count": 62, "max_ms": 109.521, "p95_ms": 97.319}, "yolo11_640_fire_11class": {"avg_ms": 76.084, "count": 62, "max_ms": 90.174, "p95_ms": 87.01}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.11, "count": 62, "max_ms": 99.608, "p95_ms": 95.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.225, "count": 62, "max_ms": 122.35, "p95_ms": 114.592}, "read_interval": {"avg_ms": 418.339, "count": 62, "max_ms": 10160.558, "p95_ms": 133.224}, "triton_wait": {"avg_ms": 85.776, "count": 62, "max_ms": 109.534, "p95_ms": 99.62}, "visualize": {"avg_ms": 3.769, "count": 62, "max_ms": 6.06, "p95_ms": 5.835}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6152.008, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55bc30390180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bc30390180] error while decoding MB 0 17, bytestream 69067 [h264 @ 0x55bc3029c380] error while decoding MB 26 6, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 130, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:34:43+09:00", "last_error_summary": "[h264 @ 0x55bc30390180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bc30390180] error while decoding MB 0 17, bytestream 69067 [h264 @ 0x55bc3029c380] error while decoding MB 26 6, bytestream -7", "last_exit_at": "2026-07-06T23:26:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 498.725, "last_started_at": "2026-07-06T23:26:24+09:00", "pid": 35478, "process": {"exists": false, "pid": 35478}, "restart_count": 139, "restart_count_per_hour": 8, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.119, "sample_count": 2708, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 310017, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:34:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-06T23:34:43+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T23:34:43+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-06T23:34:43.395241+09:00", "mtime_epoch": 1783348483.3952415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119009}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:34:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1686.777, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 270, "pushed_frames": 270, "read_frames": 270, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.01, "pushed": 9.01, "read": 9.01}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.593, "count": 270, "max_ms": 6.091, "p95_ms": 5.355}, "gstreamer_push": {"avg_ms": 1.379, "count": 270, "max_ms": 2.459, "p95_ms": 1.722}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.07, "count": 270, "max_ms": 93.503, "p95_ms": 85.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.319, "count": 270, "max_ms": 118.408, "p95_ms": 102.443}, "yolo11_640_fire_11class": {"avg_ms": 75.628, "count": 270, "max_ms": 94.152, "p95_ms": 87.082}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.824, "count": 270, "max_ms": 103.856, "p95_ms": 98.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.748, "count": 270, "max_ms": 130.976, "p95_ms": 115.695}, "read_interval": {"avg_ms": 111.212, "count": 270, "max_ms": 144.166, "p95_ms": 129.892}, "triton_wait": {"avg_ms": 85.192, "count": 270, "max_ms": 118.42, "p95_ms": 102.89}, "visualize": {"avg_ms": 3.731, "count": 270, "max_ms": 6.369, "p95_ms": 4.696}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6151.441, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:34:43+09:00", "last_error_summary": "[h264 @ 0x55bc30390180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bc30390180] error while decoding MB 0 17, bytestream 69067 [h264 @ 0x55bc3029c380] error while decoding MB 26 6, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 130, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:34:43+09:00", "last_error_summary": "[h264 @ 0x55bc30390180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bc30390180] error while decoding MB 0 17, bytestream 69067 [h264 @ 0x55bc3029c380] error while decoding MB 26 6, bytestream -7", "last_exit_at": "2026-07-06T23:34:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 498.797, "last_started_at": "2026-07-06T23:26:24+09:00", "pid": 35478, "process": {"exists": false, "pid": 35478}, "restart_count": 139, "restart_count_per_hour": 8, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.119, "sample_count": 2708, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 310018, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:34:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T23:34:43+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-06T23:34:43+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-06T23:34:43.395241+09:00", "mtime_epoch": 1783348483.3952415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119009}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:34:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1686.777, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 270, "pushed_frames": 270, "read_frames": 269, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.021, "pushed": 9.021, "read": 8.988}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.593, "count": 270, "max_ms": 6.091, "p95_ms": 5.355}, "gstreamer_push": {"avg_ms": 1.379, "count": 270, "max_ms": 2.459, "p95_ms": 1.722}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.056, "count": 270, "max_ms": 93.503, "p95_ms": 85.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.368, "count": 270, "max_ms": 118.408, "p95_ms": 102.555}, "yolo11_640_fire_11class": {"avg_ms": 75.621, "count": 270, "max_ms": 94.152, "p95_ms": 87.082}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.783, "count": 270, "max_ms": 103.856, "p95_ms": 97.941}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.754, "count": 270, "max_ms": 130.976, "p95_ms": 115.695}, "read_interval": {"avg_ms": 111.154, "count": 269, "max_ms": 144.166, "p95_ms": 129.892}, "triton_wait": {"avg_ms": 85.199, "count": 270, "max_ms": 118.42, "p95_ms": 102.972}, "visualize": {"avg_ms": 3.731, "count": 270, "max_ms": 6.369, "p95_ms": 4.696}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6151.629, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 130, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:34:43+09:00", "last_error_summary": "[h264 @ 0x55bc30390180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bc30390180] error while decoding MB 0 17, bytestream 69067 [h264 @ 0x55bc3029c380] error while decoding MB 26 6, bytestream -7", "last_exit_at": "2026-07-06T23:34:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 498.806, "last_started_at": "2026-07-06T23:26:24+09:00", "pid": 35478, "process": {"exists": false, "pid": 35478}, "restart_count": 139, "restart_count_per_hour": 8, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.119, "sample_count": 2708, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 310018, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:34:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-06T23:34:43+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-06T23:34:43+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-06T23:34:43.395241+09:00", "mtime_epoch": 1783348483.3952415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119009}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:34:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1686.777, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 270, "pushed_frames": 270, "read_frames": 269, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.018, "pushed": 9.018, "read": 8.985}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.593, "count": 270, "max_ms": 6.091, "p95_ms": 5.355}, "gstreamer_push": {"avg_ms": 1.379, "count": 270, "max_ms": 2.459, "p95_ms": 1.722}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.056, "count": 270, "max_ms": 93.503, "p95_ms": 85.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.368, "count": 270, "max_ms": 118.408, "p95_ms": 102.555}, "yolo11_640_fire_11class": {"avg_ms": 75.621, "count": 270, "max_ms": 94.152, "p95_ms": 87.082}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.783, "count": 270, "max_ms": 103.856, "p95_ms": 97.941}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.754, "count": 270, "max_ms": 130.976, "p95_ms": 115.695}, "read_interval": {"avg_ms": 111.154, "count": 269, "max_ms": 144.166, "p95_ms": 129.892}, "triton_wait": {"avg_ms": 85.199, "count": 270, "max_ms": 118.42, "p95_ms": 102.972}, "visualize": {"avg_ms": 3.731, "count": 270, "max_ms": 6.369, "p95_ms": 4.696}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6258.215, "limit_mb": 8796093022207.996}, "detail": {"pid": 35737, "restart_count": 140, "start_count": 141}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 130, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:34:43+09:00", "last_error_summary": "[h264 @ 0x55bc30390180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bc30390180] error while decoding MB 0 17, bytestream 69067 [h264 @ 0x55bc3029c380] error while decoding MB 26 6, bytestream -7", "last_exit_at": "2026-07-06T23:34:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:34:45+09:00", "pid": 35737, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.609, "name": "ffmpeg", "pid": 35737, "rss_mb": 138.609, "state": "S (sleeping)", "threads": 106}, "restart_count": 140, "restart_count_per_hour": 9, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.119, "sample_count": 2672, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 310018, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:34:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.103, "last_frame_pushed_at": "2026-07-06T23:34:43+09:00", "last_frame_read_age_sec": 4.22, "last_frame_read_at": "2026-07-06T23:34:43+09:00", "mjpeg": {"age_sec": 4.085, "exists": true, "mtime": "2026-07-06T23:34:43.519243+09:00", "mtime_epoch": 1783348483.5192435, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118920}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:34:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1686.777, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 233, "pushed_frames": 233, "read_frames": 232, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.785, "pushed": 7.785, "read": 7.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.598, "count": 233, "max_ms": 6.091, "p95_ms": 5.689}, "gstreamer_push": {"avg_ms": 1.388, "count": 233, "max_ms": 2.459, "p95_ms": 2.146}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.874, "count": 233, "max_ms": 93.503, "p95_ms": 85.534}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.111, "count": 233, "max_ms": 118.408, "p95_ms": 102.555}, "yolo11_640_fire_11class": {"avg_ms": 75.295, "count": 233, "max_ms": 94.152, "p95_ms": 87.778}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.679, "count": 233, "max_ms": 103.856, "p95_ms": 98.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.832, "count": 233, "max_ms": 130.976, "p95_ms": 116.265}, "read_interval": {"avg_ms": 111.245, "count": 232, "max_ms": 144.166, "p95_ms": 130.624}, "triton_wait": {"avg_ms": 85.217, "count": 233, "max_ms": 118.42, "p95_ms": 103.285}, "visualize": {"avg_ms": 3.749, "count": 233, "max_ms": 6.369, "p95_ms": 6.055}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2764}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6117.414, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x558ac4cef200] missing picture in access unit with size 18 [h264 @ 0x558ac4d50400] no frame! [in#0/rtsp @ 0x558ac4ceae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558ac4ceae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558ac4cef840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:40:06+09:00", "last_error_summary": "[NULL @ 0x558ac4cef200] missing picture in access unit with size 18 [h264 @ 0x558ac4d50400] no frame! [in#0/rtsp @ 0x558ac4ceae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558ac4ceae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558ac4cef840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T23:33:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 397.677, "last_started_at": "2026-07-06T23:33:28+09:00", "pid": 35585, "process": {"exists": false, "pid": 35585}, "restart_count": 90, "restart_count_per_hour": 11, "rtsp_timeout_count": 144, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.031, "sample_count": 2386, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 331767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:40:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-06T23:40:06+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-06T23:40:06+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-06T23:40:06.595959+09:00", "mtime_epoch": 1783348806.595959, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 114352}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:40:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1570.812, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.469, "pushed": 0.469, "read": 0.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.231, "count": 13, "max_ms": 6.23, "p95_ms": 6.23}, "gstreamer_push": {"avg_ms": 1.294, "count": 13, "max_ms": 1.422, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.001, "count": 13, "max_ms": 64.635, "p95_ms": 64.635}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.701, "count": 13, "max_ms": 83.856, "p95_ms": 83.856}, "yolo11_640_fire_11class": {"avg_ms": 58.416, "count": 13, "max_ms": 70.314, "p95_ms": 70.314}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.226, "count": 13, "max_ms": 71.035, "p95_ms": 71.035}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.019, "count": 13, "max_ms": 96.293, "p95_ms": 96.293}, "read_interval": {"avg_ms": 2350.814, "count": 14, "max_ms": 16588.669, "p95_ms": 16588.669}, "triton_wait": {"avg_ms": 63.203, "count": 13, "max_ms": 83.869, "p95_ms": 83.869}, "visualize": {"avg_ms": 3.675, "count": 13, "max_ms": 4.587, "p95_ms": 4.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.733, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2764}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6116.57, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:40:06+09:00", "last_error_summary": "[NULL @ 0x558ac4cef200] missing picture in access unit with size 18 [h264 @ 0x558ac4d50400] no frame! [in#0/rtsp @ 0x558ac4ceae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558ac4ceae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558ac4cef840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:40:06+09:00", "last_error_summary": "[NULL @ 0x558ac4cef200] missing picture in access unit with size 18 [h264 @ 0x558ac4d50400] no frame! [in#0/rtsp @ 0x558ac4ceae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558ac4ceae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558ac4cef840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T23:40:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 397.719, "last_started_at": "2026-07-06T23:33:28+09:00", "pid": 35585, "process": {"exists": false, "pid": 35585}, "restart_count": 90, "restart_count_per_hour": 11, "rtsp_timeout_count": 144, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.031, "sample_count": 2386, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 331768, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:40:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-06T23:40:06+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-06T23:40:06+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-06T23:40:06.595959+09:00", "mtime_epoch": 1783348806.595959, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 114352}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:40:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1570.812, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.504, "pushed": 0.504, "read": 0.504}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.213, "count": 14, "max_ms": 6.23, "p95_ms": 6.23}, "gstreamer_push": {"avg_ms": 1.301, "count": 14, "max_ms": 1.422, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.493, "count": 14, "max_ms": 64.635, "p95_ms": 64.635}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.194, "count": 14, "max_ms": 83.856, "p95_ms": 83.856}, "yolo11_640_fire_11class": {"avg_ms": 59.106, "count": 14, "max_ms": 70.314, "p95_ms": 70.314}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.858, "count": 14, "max_ms": 71.035, "p95_ms": 71.035}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.463, "count": 14, "max_ms": 96.293, "p95_ms": 96.293}, "read_interval": {"avg_ms": 2350.814, "count": 14, "max_ms": 16588.669, "p95_ms": 16588.669}, "triton_wait": {"avg_ms": 64.66, "count": 14, "max_ms": 83.869, "p95_ms": 83.869}, "visualize": {"avg_ms": 3.68, "count": 14, "max_ms": 4.587, "p95_ms": 4.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.775, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2764}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6116.562, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:40:06+09:00", "last_error_summary": "[NULL @ 0x558ac4cef200] missing picture in access unit with size 18 [h264 @ 0x558ac4d50400] no frame! [in#0/rtsp @ 0x558ac4ceae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558ac4ceae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558ac4cef840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T23:40:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 397.729, "last_started_at": "2026-07-06T23:33:28+09:00", "pid": 35585, "process": {"exists": false, "pid": 35585}, "restart_count": 90, "restart_count_per_hour": 11, "rtsp_timeout_count": 144, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.031, "sample_count": 2386, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 331768, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:40:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-06T23:40:06+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-06T23:40:06+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-06T23:40:06.595959+09:00", "mtime_epoch": 1783348806.595959, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 114352}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:40:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1570.812, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.504, "pushed": 0.504, "read": 0.504}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.213, "count": 14, "max_ms": 6.23, "p95_ms": 6.23}, "gstreamer_push": {"avg_ms": 1.301, "count": 14, "max_ms": 1.422, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.493, "count": 14, "max_ms": 64.635, "p95_ms": 64.635}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.194, "count": 14, "max_ms": 83.856, "p95_ms": 83.856}, "yolo11_640_fire_11class": {"avg_ms": 59.106, "count": 14, "max_ms": 70.314, "p95_ms": 70.314}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.858, "count": 14, "max_ms": 71.035, "p95_ms": 71.035}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.463, "count": 14, "max_ms": 96.293, "p95_ms": 96.293}, "read_interval": {"avg_ms": 2350.814, "count": 14, "max_ms": 16588.669, "p95_ms": 16588.669}, "triton_wait": {"avg_ms": 64.66, "count": 14, "max_ms": 83.869, "p95_ms": 83.869}, "visualize": {"avg_ms": 3.68, "count": 14, "max_ms": 4.587, "p95_ms": 4.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.785, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5876.648, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5598883a5440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5598883a5440] error while decoding MB 0 15, bytestream 37 [h264 @ 0x5598877371c0] cabac decode of qscale diff failed at 5 28 [h264 @ 0x5598877371c0] error while decoding MB 5 28, bytestream 7043 [h264 @ 0x5598884742c0] error while decoding MB 26 15, bytestream -27 [h264 @ 0x5598876e3cc0] reference picture missing during reorder [h264 @ 0x5598876e3cc0] Missing reference picture, default is 65555 [h264 @ 0x5598", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 131, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:40:10+09:00", "last_error_summary": "[h264 @ 0x5598883a5440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5598883a5440] error while decoding MB 0 15, bytestream 37 [h264 @ 0x5598877371c0] cabac decode of qscale diff failed at 5 28 [h264 @ 0x5598877371c0] error while decoding MB 5 28, bytestream 7043 [h264 @ 0x5598884742c0] error while decoding MB 26 15, bytestream -27 [h264 @ 0x5598876e3cc0] reference picture missing during reorder [h264 @ 0x5598876e3cc0] Missing reference picture, default is 65555 [h264 @ 0x5598", "last_exit_at": "2026-07-06T23:34:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 325.119, "last_started_at": "2026-07-06T23:34:45+09:00", "pid": 35737, "process": {"exists": false, "pid": 35737}, "restart_count": 140, "restart_count_per_hour": 8, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.519, "sample_count": 2535, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 312762, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:40:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-06T23:40:10+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-06T23:40:10+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T23:40:10.668018+09:00", "mtime_epoch": 1783348810.6680183, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136840}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:40:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1688.066, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 118, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.941, "pushed": 3.941, "read": 3.941}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.54, "count": 118, "max_ms": 5.562, "p95_ms": 4.211}, "gstreamer_push": {"avg_ms": 1.281, "count": 118, "max_ms": 1.445, "p95_ms": 1.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.27, "count": 118, "max_ms": 70.305, "p95_ms": 57.575}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.314, "count": 118, "max_ms": 87.702, "p95_ms": 83.358}, "yolo11_640_fire_11class": {"avg_ms": 55.525, "count": 118, "max_ms": 68.641, "p95_ms": 63.734}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.139, "count": 118, "max_ms": 70.316, "p95_ms": 62.563}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.342, "count": 118, "max_ms": 110.869, "p95_ms": 96.325}, "read_interval": {"avg_ms": 253.823, "count": 118, "max_ms": 19984.174, "p95_ms": 110.316}, "triton_wait": {"avg_ms": 59.069, "count": 118, "max_ms": 87.714, "p95_ms": 83.371}, "visualize": {"avg_ms": 3.613, "count": 118, "max_ms": 5.769, "p95_ms": 3.969}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5876.332, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:40:10+09:00", "last_error_summary": "[h264 @ 0x5598883a5440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5598883a5440] error while decoding MB 0 15, bytestream 37 [h264 @ 0x5598877371c0] cabac decode of qscale diff failed at 5 28 [h264 @ 0x5598877371c0] error while decoding MB 5 28, bytestream 7043 [h264 @ 0x5598884742c0] error while decoding MB 26 15, bytestream -27 [h264 @ 0x5598876e3cc0] reference picture missing during reorder [h264 @ 0x5598876e3cc0] Missing reference picture, default is 65555 [h264 @ 0x5598", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 131, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:40:10+09:00", "last_error_summary": "[h264 @ 0x5598883a5440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5598883a5440] error while decoding MB 0 15, bytestream 37 [h264 @ 0x5598877371c0] cabac decode of qscale diff failed at 5 28 [h264 @ 0x5598877371c0] error while decoding MB 5 28, bytestream 7043 [h264 @ 0x5598884742c0] error while decoding MB 26 15, bytestream -27 [h264 @ 0x5598876e3cc0] reference picture missing during reorder [h264 @ 0x5598876e3cc0] Missing reference picture, default is 65555 [h264 @ 0x5598", "last_exit_at": "2026-07-06T23:40:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 325.146, "last_started_at": "2026-07-06T23:34:45+09:00", "pid": 35737, "process": {"exists": false, "pid": 35737}, "restart_count": 140, "restart_count_per_hour": 8, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.519, "sample_count": 2535, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 312763, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:40:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T23:40:10+09:00", "last_frame_read_age_sec": 0.083, "last_frame_read_at": "2026-07-06T23:40:10+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-06T23:40:10.668018+09:00", "mtime_epoch": 1783348810.6680183, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136840}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:40:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1688.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.971, "pushed": 3.971, "read": 3.938}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.589, "count": 119, "max_ms": 9.463, "p95_ms": 4.987}, "gstreamer_push": {"avg_ms": 1.282, "count": 119, "max_ms": 1.445, "p95_ms": 1.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.243, "count": 119, "max_ms": 70.305, "p95_ms": 57.575}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.281, "count": 119, "max_ms": 87.702, "p95_ms": 83.358}, "yolo11_640_fire_11class": {"avg_ms": 55.526, "count": 119, "max_ms": 68.641, "p95_ms": 63.734}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.131, "count": 119, "max_ms": 70.316, "p95_ms": 62.563}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.373, "count": 119, "max_ms": 110.869, "p95_ms": 96.325}, "read_interval": {"avg_ms": 253.823, "count": 118, "max_ms": 19984.174, "p95_ms": 110.316}, "triton_wait": {"avg_ms": 59.04, "count": 119, "max_ms": 87.714, "p95_ms": 83.371}, "visualize": {"avg_ms": 3.627, "count": 119, "max_ms": 5.769, "p95_ms": 4.066}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5876.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 131, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:40:10+09:00", "last_error_summary": "[h264 @ 0x5598883a5440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5598883a5440] error while decoding MB 0 15, bytestream 37 [h264 @ 0x5598877371c0] cabac decode of qscale diff failed at 5 28 [h264 @ 0x5598877371c0] error while decoding MB 5 28, bytestream 7043 [h264 @ 0x5598884742c0] error while decoding MB 26 15, bytestream -27 [h264 @ 0x5598876e3cc0] reference picture missing during reorder [h264 @ 0x5598876e3cc0] Missing reference picture, default is 65555 [h264 @ 0x5598", "last_exit_at": "2026-07-06T23:40:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 325.154, "last_started_at": "2026-07-06T23:34:45+09:00", "pid": 35737, "process": {"exists": false, "pid": 35737}, "restart_count": 140, "restart_count_per_hour": 8, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.519, "sample_count": 2535, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 312763, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:40:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-06T23:40:10+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-06T23:40:10+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-06T23:40:10.668018+09:00", "mtime_epoch": 1783348810.6680183, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136840}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:40:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1688.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.97, "pushed": 3.97, "read": 3.937}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.589, "count": 119, "max_ms": 9.463, "p95_ms": 4.987}, "gstreamer_push": {"avg_ms": 1.282, "count": 119, "max_ms": 1.445, "p95_ms": 1.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.243, "count": 119, "max_ms": 70.305, "p95_ms": 57.575}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.281, "count": 119, "max_ms": 87.702, "p95_ms": 83.358}, "yolo11_640_fire_11class": {"avg_ms": 55.526, "count": 119, "max_ms": 68.641, "p95_ms": 63.734}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.131, "count": 119, "max_ms": 70.316, "p95_ms": 62.563}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.373, "count": 119, "max_ms": 110.869, "p95_ms": 96.325}, "read_interval": {"avg_ms": 253.823, "count": 118, "max_ms": 19984.174, "p95_ms": 110.316}, "triton_wait": {"avg_ms": 59.04, "count": 119, "max_ms": 87.714, "p95_ms": 83.371}, "visualize": {"avg_ms": 3.627, "count": 119, "max_ms": 5.769, "p95_ms": 4.066}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2764}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6034.504, "limit_mb": 8796093022207.996}, "detail": {"pid": 35844, "restart_count": 91, "start_count": 92}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:40:06+09:00", "last_error_summary": "[NULL @ 0x558ac4cef200] missing picture in access unit with size 18 [h264 @ 0x558ac4d50400] no frame! [in#0/rtsp @ 0x558ac4ceae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558ac4ceae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558ac4cef840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T23:40:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:40:09+09:00", "pid": 35844, "process": {"exists": true, "fd_count": 5, "hwm_mb": 208.941, "name": "ffmpeg", "pid": 35844, "rss_mb": 208.941, "state": "S (sleeping)", "threads": 106}, "restart_count": 91, "restart_count_per_hour": 12, "rtsp_timeout_count": 144, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.069, "sample_count": 2343, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 331768, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:40:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.884, "last_frame_pushed_at": "2026-07-06T23:40:06+09:00", "last_frame_read_age_sec": 4.983, "last_frame_read_at": "2026-07-06T23:40:06+09:00", "mjpeg": {"age_sec": 4.865, "exists": true, "mtime": "2026-07-06T23:40:06.707961+09:00", "mtime_epoch": 1783348806.7079606, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:40:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1570.812, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.577, "pushed": 0.577, "read": 0.577}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.221, "count": 13, "max_ms": 6.23, "p95_ms": 6.23}, "gstreamer_push": {"avg_ms": 1.307, "count": 13, "max_ms": 1.422, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.406, "count": 13, "max_ms": 58.5, "p95_ms": 58.5}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.724, "count": 13, "max_ms": 83.856, "p95_ms": 83.856}, "yolo11_640_fire_11class": {"avg_ms": 58.244, "count": 13, "max_ms": 68.975, "p95_ms": 68.975}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.998, "count": 13, "max_ms": 68.986, "p95_ms": 68.986}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.992, "count": 13, "max_ms": 96.293, "p95_ms": 96.293}, "read_interval": {"avg_ms": 2128.355, "count": 13, "max_ms": 16588.669, "p95_ms": 16588.669}, "triton_wait": {"avg_ms": 64.169, "count": 13, "max_ms": 83.869, "p95_ms": 83.869}, "visualize": {"avg_ms": 3.684, "count": 13, "max_ms": 4.587, "p95_ms": 4.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.534, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6213.926, "limit_mb": 8796093022207.996}, "detail": {"pid": 35996, "restart_count": 141, "start_count": 142}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 131, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:40:10+09:00", "last_error_summary": "[h264 @ 0x5598883a5440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5598883a5440] error while decoding MB 0 15, bytestream 37 [h264 @ 0x5598877371c0] cabac decode of qscale diff failed at 5 28 [h264 @ 0x5598877371c0] error while decoding MB 5 28, bytestream 7043 [h264 @ 0x5598884742c0] error while decoding MB 26 15, bytestream -27 [h264 @ 0x5598876e3cc0] reference picture missing during reorder [h264 @ 0x5598876e3cc0] Missing reference picture, default is 65555 [h264 @ 0x5598", "last_exit_at": "2026-07-06T23:40:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:40:13+09:00", "pid": 35996, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.387, "name": "ffmpeg", "pid": 35996, "rss_mb": 139.387, "state": "S (sleeping)", "threads": 106}, "restart_count": 141, "restart_count_per_hour": 9, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.532, "sample_count": 2494, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 312763, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:40:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.608, "last_frame_pushed_at": "2026-07-06T23:40:10+09:00", "last_frame_read_age_sec": 4.685, "last_frame_read_at": "2026-07-06T23:40:10+09:00", "mjpeg": {"age_sec": 4.591, "exists": true, "mtime": "2026-07-06T23:40:10.760020+09:00", "mtime_epoch": 1783348810.7600198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 137804}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:40:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1688.066, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.169, "pushed": 2.169, "read": 2.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.593, "count": 65, "max_ms": 9.463, "p95_ms": 3.862}, "gstreamer_push": {"avg_ms": 1.29, "count": 65, "max_ms": 1.445, "p95_ms": 1.39}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.851, "count": 65, "max_ms": 70.305, "p95_ms": 57.575}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.832, "count": 65, "max_ms": 87.702, "p95_ms": 84.13}, "yolo11_640_fire_11class": {"avg_ms": 55.803, "count": 65, "max_ms": 68.641, "p95_ms": 65.925}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.541, "count": 65, "max_ms": 70.316, "p95_ms": 65.912}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.37, "count": 65, "max_ms": 99.123, "p95_ms": 96.325}, "read_interval": {"avg_ms": 396.008, "count": 64, "max_ms": 19984.174, "p95_ms": 110.316}, "triton_wait": {"avg_ms": 60.378, "count": 65, "max_ms": 87.714, "p95_ms": 84.142}, "visualize": {"avg_ms": 3.601, "count": 65, "max_ms": 5.209, "p95_ms": 3.747}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2776}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6096.703, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55cce3574e80] error while decoding MB 95 24, bytestream -11 [in#0/rtsp @ 0x55cce2fcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cce2fcde00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:43:20+09:00", "last_error_summary": "[h264 @ 0x55cce3574e80] error while decoding MB 95 24, bytestream -11 [in#0/rtsp @ 0x55cce2fcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cce2fcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:40:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.457, "last_started_at": "2026-07-06T23:40:09+09:00", "pid": 35844, "process": {"exists": false, "pid": 35844}, "restart_count": 91, "restart_count_per_hour": 11, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.394, "sample_count": 2074, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 333177, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:43:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.086, "last_frame_pushed_at": "2026-07-06T23:43:19+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-06T23:43:19+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-06T23:43:19.958782+09:00", "mtime_epoch": 1783348999.9587824, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115971}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:43:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1571.586, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.728, "pushed": 0.728, "read": 0.784}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.921, "count": 13, "max_ms": 7.797, "p95_ms": 7.797}, "gstreamer_push": {"avg_ms": 1.371, "count": 13, "max_ms": 1.783, "p95_ms": 1.783}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.219, "count": 13, "max_ms": 79.191, "p95_ms": 79.191}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.64, "count": 13, "max_ms": 113.497, "p95_ms": 113.497}, "yolo11_640_fire_11class": {"avg_ms": 69.711, "count": 13, "max_ms": 86.023, "p95_ms": 86.023}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.742, "count": 13, "max_ms": 90.299, "p95_ms": 90.299}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.331, "count": 13, "max_ms": 126.475, "p95_ms": 126.475}, "read_interval": {"avg_ms": 2378.296, "count": 14, "max_ms": 16558.517, "p95_ms": 16558.517}, "triton_wait": {"avg_ms": 77.915, "count": 13, "max_ms": 113.509, "p95_ms": 113.509}, "visualize": {"avg_ms": 4.134, "count": 13, "max_ms": 6.072, "p95_ms": 6.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.866, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2776}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6096.754, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:43:20+09:00", "last_error_summary": "[h264 @ 0x55cce3574e80] error while decoding MB 95 24, bytestream -11 [in#0/rtsp @ 0x55cce2fcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cce2fcde00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:43:20+09:00", "last_error_summary": "[h264 @ 0x55cce3574e80] error while decoding MB 95 24, bytestream -11 [in#0/rtsp @ 0x55cce2fcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cce2fcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:43:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.485, "last_started_at": "2026-07-06T23:40:09+09:00", "pid": 35844, "process": {"exists": false, "pid": 35844}, "restart_count": 91, "restart_count_per_hour": 11, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.394, "sample_count": 2075, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 333178, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:43:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T23:43:20+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-06T23:43:19+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-06T23:43:19.958782+09:00", "mtime_epoch": 1783348999.9587824, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115971}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:43:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1571.586, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.935, "count": 14, "max_ms": 7.797, "p95_ms": 7.797}, "gstreamer_push": {"avg_ms": 1.37, "count": 14, "max_ms": 1.783, "p95_ms": 1.783}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.637, "count": 14, "max_ms": 81.063, "p95_ms": 81.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.028, "count": 14, "max_ms": 113.497, "p95_ms": 113.497}, "yolo11_640_fire_11class": {"avg_ms": 70.125, "count": 14, "max_ms": 86.023, "p95_ms": 86.023}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.389, "count": 14, "max_ms": 90.299, "p95_ms": 90.299}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.793, "count": 14, "max_ms": 126.475, "p95_ms": 126.475}, "read_interval": {"avg_ms": 2378.296, "count": 14, "max_ms": 16558.517, "p95_ms": 16558.517}, "triton_wait": {"avg_ms": 78.408, "count": 14, "max_ms": 113.509, "p95_ms": 113.509}, "visualize": {"avg_ms": 4.099, "count": 14, "max_ms": 6.072, "p95_ms": 6.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.894, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2776}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6096.754, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:43:20+09:00", "last_error_summary": "[h264 @ 0x55cce3574e80] error while decoding MB 95 24, bytestream -11 [in#0/rtsp @ 0x55cce2fcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cce2fcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:43:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.492, "last_started_at": "2026-07-06T23:40:09+09:00", "pid": 35844, "process": {"exists": false, "pid": 35844}, "restart_count": 91, "restart_count_per_hour": 11, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 117.394, "sample_count": 2075, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 333178, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:43:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T23:43:20+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-06T23:43:19+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-06T23:43:19.958782+09:00", "mtime_epoch": 1783348999.9587824, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 115971}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:43:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1571.586, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.935, "count": 14, "max_ms": 7.797, "p95_ms": 7.797}, "gstreamer_push": {"avg_ms": 1.37, "count": 14, "max_ms": 1.783, "p95_ms": 1.783}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.637, "count": 14, "max_ms": 81.063, "p95_ms": 81.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.028, "count": 14, "max_ms": 113.497, "p95_ms": 113.497}, "yolo11_640_fire_11class": {"avg_ms": 70.125, "count": 14, "max_ms": 86.023, "p95_ms": 86.023}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.389, "count": 14, "max_ms": 90.299, "p95_ms": 90.299}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.793, "count": 14, "max_ms": 126.475, "p95_ms": 126.475}, "read_interval": {"avg_ms": 2378.296, "count": 14, "max_ms": 16558.517, "p95_ms": 16558.517}, "triton_wait": {"avg_ms": 78.408, "count": 14, "max_ms": 113.509, "p95_ms": 113.509}, "visualize": {"avg_ms": 4.099, "count": 14, "max_ms": 6.072, "p95_ms": 6.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.901, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2776}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6114.527, "limit_mb": 8796093022207.996}, "detail": {"pid": 36103, "restart_count": 92, "start_count": 93}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:43:20+09:00", "last_error_summary": "[h264 @ 0x55cce3574e80] error while decoding MB 95 24, bytestream -11 [in#0/rtsp @ 0x55cce2fcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cce2fcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:43:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:43:22+09:00", "pid": 36103, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.234, "name": "ffmpeg", "pid": 36103, "rss_mb": 58.234, "state": "S (sleeping)", "threads": 1}, "restart_count": 92, "restart_count_per_hour": 12, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.953, "sample_count": 2031, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 333178, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:43:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.866, "last_frame_pushed_at": "2026-07-06T23:43:20+09:00", "last_frame_read_age_sec": 4.967, "last_frame_read_at": "2026-07-06T23:43:19+09:00", "mjpeg": {"age_sec": 4.851, "exists": true, "mtime": "2026-07-06T23:43:20.066784+09:00", "mtime_epoch": 1783349000.0667841, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 110489}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:43:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1571.586, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.615, "pushed": 0.615, "read": 0.615}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.935, "count": 14, "max_ms": 7.797, "p95_ms": 7.797}, "gstreamer_push": {"avg_ms": 1.37, "count": 14, "max_ms": 1.783, "p95_ms": 1.783}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.637, "count": 14, "max_ms": 81.063, "p95_ms": 81.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.028, "count": 14, "max_ms": 113.497, "p95_ms": 113.497}, "yolo11_640_fire_11class": {"avg_ms": 70.125, "count": 14, "max_ms": 86.023, "p95_ms": 86.023}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.389, "count": 14, "max_ms": 90.299, "p95_ms": 90.299}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.793, "count": 14, "max_ms": 126.475, "p95_ms": 126.475}, "read_interval": {"avg_ms": 2378.296, "count": 14, "max_ms": 16558.517, "p95_ms": 16558.517}, "triton_wait": {"avg_ms": 78.408, "count": 14, "max_ms": 113.509, "p95_ms": 113.509}, "visualize": {"avg_ms": 4.099, "count": 14, "max_ms": 6.072, "p95_ms": 6.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.754, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2652}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6165.648, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561cfcbc2e00] error while decoding MB 27 12, bytestream -9 [h264 @ 0x561cfcc91c80] cabac decode of qscale diff failed at 35 30 [h264 @ 0x561cfcc91c80] error while decoding MB 35 30, bytestream 5261 [in#0/rtsp @ 0x561cfbfa6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561cfbfa6e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 132, "h264_decode_error_count": 442, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:50:46+09:00", "last_error_summary": "[h264 @ 0x561cfcbc2e00] error while decoding MB 27 12, bytestream -9 [h264 @ 0x561cfcc91c80] cabac decode of qscale diff failed at 35 30 [h264 @ 0x561cfcc91c80] error while decoding MB 35 30, bytestream 5261 [in#0/rtsp @ 0x561cfbfa6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561cfbfa6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:40:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 632.911, "last_started_at": "2026-07-06T23:40:13+09:00", "pid": 35996, "process": {"exists": false, "pid": 35996}, "restart_count": 141, "restart_count_per_hour": 8, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.451, "sample_count": 2501, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318319, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:50:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-06T23:50:46+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-06T23:50:46+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-06T23:50:46.215435+09:00", "mtime_epoch": 1783349446.2154346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125029}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:50:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1690.645, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.368, "pushed": 3.368, "read": 3.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.499, "count": 101, "max_ms": 4.327, "p95_ms": 3.985}, "gstreamer_push": {"avg_ms": 1.317, "count": 101, "max_ms": 1.609, "p95_ms": 1.466}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.108, "count": 101, "max_ms": 90.661, "p95_ms": 84.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.473, "count": 101, "max_ms": 114.918, "p95_ms": 98.883}, "yolo11_640_fire_11class": {"avg_ms": 74.963, "count": 101, "max_ms": 90.679, "p95_ms": 86.522}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.744, "count": 101, "max_ms": 99.296, "p95_ms": 93.024}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.13, "count": 101, "max_ms": 127.002, "p95_ms": 111.561}, "read_interval": {"avg_ms": 294.793, "count": 102, "max_ms": 10123.269, "p95_ms": 129.945}, "triton_wait": {"avg_ms": 84.028, "count": 101, "max_ms": 114.931, "p95_ms": 99.307}, "visualize": {"avg_ms": 3.603, "count": 101, "max_ms": 4.272, "p95_ms": 3.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2652}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6165.035, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:50:46+09:00", "last_error_summary": "[h264 @ 0x561cfcbc2e00] error while decoding MB 27 12, bytestream -9 [h264 @ 0x561cfcc91c80] cabac decode of qscale diff failed at 35 30 [h264 @ 0x561cfcc91c80] error while decoding MB 35 30, bytestream 5261 [in#0/rtsp @ 0x561cfbfa6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561cfbfa6e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 132, "h264_decode_error_count": 442, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:50:46+09:00", "last_error_summary": "[h264 @ 0x561cfcbc2e00] error while decoding MB 27 12, bytestream -9 [h264 @ 0x561cfcc91c80] cabac decode of qscale diff failed at 35 30 [h264 @ 0x561cfcc91c80] error while decoding MB 35 30, bytestream 5261 [in#0/rtsp @ 0x561cfbfa6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561cfbfa6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:50:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 632.967, "last_started_at": "2026-07-06T23:40:13+09:00", "pid": 35996, "process": {"exists": false, "pid": 35996}, "restart_count": 141, "restart_count_per_hour": 8, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.451, "sample_count": 2502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318320, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:50:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-06T23:50:46+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-06T23:50:46+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-06T23:50:46.215435+09:00", "mtime_epoch": 1783349446.2154346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125029}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:50:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1690.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.405, "pushed": 3.405, "read": 3.372}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.499, "count": 102, "max_ms": 4.327, "p95_ms": 3.985}, "gstreamer_push": {"avg_ms": 1.318, "count": 102, "max_ms": 1.609, "p95_ms": 1.466}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.034, "count": 102, "max_ms": 90.661, "p95_ms": 84.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.528, "count": 102, "max_ms": 114.918, "p95_ms": 98.883}, "yolo11_640_fire_11class": {"avg_ms": 75.007, "count": 102, "max_ms": 90.679, "p95_ms": 86.522}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.8, "count": 102, "max_ms": 99.296, "p95_ms": 93.024}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.175, "count": 102, "max_ms": 127.002, "p95_ms": 111.561}, "read_interval": {"avg_ms": 296.425, "count": 101, "max_ms": 10123.269, "p95_ms": 129.945}, "triton_wait": {"avg_ms": 84.072, "count": 102, "max_ms": 114.931, "p95_ms": 99.307}, "visualize": {"avg_ms": 3.603, "count": 102, "max_ms": 4.272, "p95_ms": 3.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2652}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6165.141, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 132, "h264_decode_error_count": 442, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:50:46+09:00", "last_error_summary": "[h264 @ 0x561cfcbc2e00] error while decoding MB 27 12, bytestream -9 [h264 @ 0x561cfcc91c80] cabac decode of qscale diff failed at 35 30 [h264 @ 0x561cfcc91c80] error while decoding MB 35 30, bytestream 5261 [in#0/rtsp @ 0x561cfbfa6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561cfbfa6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:50:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 632.975, "last_started_at": "2026-07-06T23:40:13+09:00", "pid": 35996, "process": {"exists": false, "pid": 35996}, "restart_count": 141, "restart_count_per_hour": 8, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.451, "sample_count": 2502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318320, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:50:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-06T23:50:46+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T23:50:46+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-06T23:50:46.215435+09:00", "mtime_epoch": 1783349446.2154346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125029}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:50:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1690.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.404, "pushed": 3.404, "read": 3.371}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.499, "count": 102, "max_ms": 4.327, "p95_ms": 3.985}, "gstreamer_push": {"avg_ms": 1.318, "count": 102, "max_ms": 1.609, "p95_ms": 1.466}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.034, "count": 102, "max_ms": 90.661, "p95_ms": 84.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.528, "count": 102, "max_ms": 114.918, "p95_ms": 98.883}, "yolo11_640_fire_11class": {"avg_ms": 75.007, "count": 102, "max_ms": 90.679, "p95_ms": 86.522}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.8, "count": 102, "max_ms": 99.296, "p95_ms": 93.024}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.175, "count": 102, "max_ms": 127.002, "p95_ms": 111.561}, "read_interval": {"avg_ms": 296.425, "count": 101, "max_ms": 10123.269, "p95_ms": 129.945}, "triton_wait": {"avg_ms": 84.072, "count": 102, "max_ms": 114.931, "p95_ms": 99.307}, "visualize": {"avg_ms": 3.603, "count": 102, "max_ms": 4.272, "p95_ms": 3.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260706_124204_6d2894f3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 2652}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6272.617, "limit_mb": 8796093022207.996}, "detail": {"pid": 36255, "restart_count": 142, "start_count": 143}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 132, "h264_decode_error_count": 442, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:50:46+09:00", "last_error_summary": "[h264 @ 0x561cfcbc2e00] error while decoding MB 27 12, bytestream -9 [h264 @ 0x561cfcc91c80] cabac decode of qscale diff failed at 35 30 [h264 @ 0x561cfcc91c80] error while decoding MB 35 30, bytestream 5261 [in#0/rtsp @ 0x561cfbfa6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561cfbfa6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:50:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:50:48+09:00", "pid": 36255, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.07, "name": "ffmpeg", "pid": 36255, "rss_mb": 139.07, "state": "S (sleeping)", "threads": 106}, "restart_count": 142, "restart_count_per_hour": 9, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.423, "sample_count": 2463, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318320, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:50:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "6d2894f3", "last_frame_pushed_age_sec": 4.339, "last_frame_pushed_at": "2026-07-06T23:50:46+09:00", "last_frame_read_age_sec": 4.442, "last_frame_read_at": "2026-07-06T23:50:46+09:00", "mjpeg": {"age_sec": 4.323, "exists": true, "mtime": "2026-07-06T23:50:46.327442+09:00", "mtime_epoch": 1783349446.3274424, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124732}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-06T23:50:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1829.199, "name": "celery", "pid": 237, "rss_mb": 1690.645, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.102, "pushed": 2.102, "read": 2.069}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.476, "count": 63, "max_ms": 4.314, "p95_ms": 3.725}, "gstreamer_push": {"avg_ms": 1.317, "count": 63, "max_ms": 1.609, "p95_ms": 1.486}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.079, "count": 63, "max_ms": 87.925, "p95_ms": 81.664}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.336, "count": 63, "max_ms": 114.918, "p95_ms": 98.883}, "yolo11_640_fire_11class": {"avg_ms": 73.453, "count": 63, "max_ms": 87.965, "p95_ms": 83.902}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.792, "count": 63, "max_ms": 99.296, "p95_ms": 93.898}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.239, "count": 63, "max_ms": 127.002, "p95_ms": 111.561}, "read_interval": {"avg_ms": 413.175, "count": 62, "max_ms": 10123.269, "p95_ms": 129.945}, "triton_wait": {"avg_ms": 83.201, "count": 63, "max_ms": 114.931, "p95_ms": 99.307}, "visualize": {"avg_ms": 3.588, "count": 63, "max_ms": 4.272, "p95_ms": 3.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-06T12:42:04+09:00", "state": "running", "task_id": "0fd5b277-dee4-47bc-98c0-d547c4de7b0d", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2813}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6403.906, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:43:20+09:00", "last_error_summary": "[h264 @ 0x55cce3574e80] error while decoding MB 95 24, bytestream -11 [in#0/rtsp @ 0x55cce2fcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cce2fcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:43:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 524.042, "last_started_at": "2026-07-06T23:43:22+09:00", "pid": 36103, "process": {"exists": true, "fd_count": 5, "hwm_mb": 332.18, "name": "ffmpeg", "pid": 36103, "rss_mb": 332.18, "state": "S (sleeping)", "threads": 106}, "restart_count": 92, "restart_count_per_hour": 10, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.97, "sample_count": 2458, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 337616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:51:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.059, "last_frame_pushed_at": "2026-07-06T23:51:36+09:00", "last_frame_read_age_sec": 30.165, "last_frame_read_at": "2026-07-06T23:51:36+09:00", "mjpeg": {"age_sec": 30.04, "exists": true, "mtime": "2026-07-06T23:51:36.918886+09:00", "mtime_epoch": 1783349496.9188864, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 109360}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:52:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1573.648, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2813}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6393.844, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:43:20+09:00", "last_error_summary": "[h264 @ 0x55cce3574e80] error while decoding MB 95 24, bytestream -11 [in#0/rtsp @ 0x55cce2fcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55cce2fcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:43:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 524.055, "last_started_at": "2026-07-06T23:43:22+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 92, "restart_count_per_hour": 10, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.97, "sample_count": 2458, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 337616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:51:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.071, "last_frame_pushed_at": "2026-07-06T23:51:36+09:00", "last_frame_read_age_sec": 30.177, "last_frame_read_at": "2026-07-06T23:51:36+09:00", "mjpeg": {"age_sec": 30.052, "exists": true, "mtime": "2026-07-06T23:51:36.918886+09:00", "mtime_epoch": 1783349496.9188864, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 109360}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:52:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1573.648, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2813}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6102.395, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557ef348e800] error while decoding MB 1 41, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 93, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:52:07+09:00", "last_error_summary": "[h264 @ 0x557ef348e800] error while decoding MB 1 41, bytestream -5", "last_exit_at": "2026-07-06T23:43:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 524.105, "last_started_at": "2026-07-06T23:43:22+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 92, "restart_count_per_hour": 10, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.97, "sample_count": 2458, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 337616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:51:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 30.122, "last_frame_pushed_at": "2026-07-06T23:51:36+09:00", "last_frame_read_age_sec": 30.228, "last_frame_read_at": "2026-07-06T23:51:36+09:00", "mjpeg": {"age_sec": 30.103, "exists": true, "mtime": "2026-07-06T23:51:36.918886+09:00", "mtime_epoch": 1783349496.9188864, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 109360}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:52:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1573.648, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2813}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6275.387, "limit_mb": 8796093022207.996}, "detail": {"pid": 36362, "restart_count": 93, "start_count": 94}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 93, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:52:07+09:00", "last_error_summary": "[h264 @ 0x557ef348e800] error while decoding MB 1 41, bytestream -5", "last_exit_at": "2026-07-06T23:43:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:52:09+09:00", "pid": 36362, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.527, "name": "ffmpeg", "pid": 36362, "rss_mb": 209.527, "state": "S (sleeping)", "threads": 106}, "restart_count": 93, "restart_count_per_hour": 11, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.009, "sample_count": 2419, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 337616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:51:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 34.53, "last_frame_pushed_at": "2026-07-06T23:51:36+09:00", "last_frame_read_age_sec": 34.636, "last_frame_read_at": "2026-07-06T23:51:36+09:00", "mjpeg": {"age_sec": 34.511, "exists": true, "mtime": "2026-07-06T23:51:36.918886+09:00", "mtime_epoch": 1783349496.9188864, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 109360}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:52:11+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1573.648, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2822}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6106.07, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5644c8db99c0] error while decoding MB 41 36, bytestream -5 [in#0/rtsp @ 0x5644c8740e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5644c8740e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:54:38+09:00", "last_error_summary": "[h264 @ 0x5644c8db99c0] error while decoding MB 41 36, bytestream -5 [in#0/rtsp @ 0x5644c8740e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5644c8740e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:43:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.459, "last_started_at": "2026-07-06T23:52:09+09:00", "pid": 36362, "process": {"exists": false, "pid": 36362}, "restart_count": 93, "restart_count_per_hour": 11, "rtsp_timeout_count": 148, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.507, "sample_count": 2141, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 338654, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:54:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-06T23:54:38+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-06T23:54:38+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-06T23:54:38.833686+09:00", "mtime_epoch": 1783349678.8336856, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 111054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:54:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1574.164, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.747, "pushed": 0.747, "read": 0.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.446, "count": 13, "max_ms": 5.961, "p95_ms": 5.961}, "gstreamer_push": {"avg_ms": 1.265, "count": 13, "max_ms": 1.398, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.625, "count": 13, "max_ms": 85.822, "p95_ms": 85.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.433, "count": 13, "max_ms": 108.394, "p95_ms": 108.394}, "yolo11_640_fire_11class": {"avg_ms": 76.854, "count": 13, "max_ms": 87.152, "p95_ms": 87.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.437, "count": 13, "max_ms": 95.512, "p95_ms": 95.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.393, "count": 13, "max_ms": 121.226, "p95_ms": 121.226}, "read_interval": {"avg_ms": 2324.01, "count": 14, "max_ms": 15998.551, "p95_ms": 15998.551}, "triton_wait": {"avg_ms": 86.183, "count": 13, "max_ms": 108.407, "p95_ms": 108.407}, "visualize": {"avg_ms": 3.732, "count": 13, "max_ms": 4.685, "p95_ms": 4.685}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.4, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2822}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6105.297, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:54:38+09:00", "last_error_summary": "[h264 @ 0x5644c8db99c0] error while decoding MB 41 36, bytestream -5 [in#0/rtsp @ 0x5644c8740e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5644c8740e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:54:38+09:00", "last_error_summary": "[h264 @ 0x5644c8db99c0] error while decoding MB 41 36, bytestream -5 [in#0/rtsp @ 0x5644c8740e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5644c8740e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:54:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.51, "last_started_at": "2026-07-06T23:52:09+09:00", "pid": 36362, "process": {"exists": false, "pid": 36362}, "restart_count": 93, "restart_count_per_hour": 11, "rtsp_timeout_count": 148, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.507, "sample_count": 2141, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 338655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:54:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T23:54:38+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-06T23:54:38+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-06T23:54:38.833686+09:00", "mtime_epoch": 1783349678.8336856, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 111054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:54:38+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1574.164, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.802, "pushed": 0.802, "read": 0.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.419, "count": 14, "max_ms": 5.961, "p95_ms": 5.961}, "gstreamer_push": {"avg_ms": 1.274, "count": 14, "max_ms": 1.398, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.974, "count": 14, "max_ms": 85.822, "p95_ms": 85.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.427, "count": 14, "max_ms": 108.394, "p95_ms": 108.394}, "yolo11_640_fire_11class": {"avg_ms": 77.06, "count": 14, "max_ms": 87.152, "p95_ms": 87.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.847, "count": 14, "max_ms": 95.512, "p95_ms": 95.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.651, "count": 14, "max_ms": 121.226, "p95_ms": 121.226}, "read_interval": {"avg_ms": 2324.01, "count": 14, "max_ms": 15998.551, "p95_ms": 15998.551}, "triton_wait": {"avg_ms": 86.469, "count": 14, "max_ms": 108.407, "p95_ms": 108.407}, "visualize": {"avg_ms": 3.729, "count": 14, "max_ms": 4.685, "p95_ms": 4.685}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.451, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2822}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6105.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:54:38+09:00", "last_error_summary": "[h264 @ 0x5644c8db99c0] error while decoding MB 41 36, bytestream -5 [in#0/rtsp @ 0x5644c8740e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5644c8740e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:54:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.517, "last_started_at": "2026-07-06T23:52:09+09:00", "pid": 36362, "process": {"exists": false, "pid": 36362}, "restart_count": 93, "restart_count_per_hour": 11, "rtsp_timeout_count": 148, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.507, "sample_count": 2141, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 338655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:54:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-06T23:54:38+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-06T23:54:38+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-06T23:54:38.833686+09:00", "mtime_epoch": 1783349678.8336856, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 111054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:54:38+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1574.164, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.802, "pushed": 0.802, "read": 0.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.419, "count": 14, "max_ms": 5.961, "p95_ms": 5.961}, "gstreamer_push": {"avg_ms": 1.274, "count": 14, "max_ms": 1.398, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.974, "count": 14, "max_ms": 85.822, "p95_ms": 85.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.427, "count": 14, "max_ms": 108.394, "p95_ms": 108.394}, "yolo11_640_fire_11class": {"avg_ms": 77.06, "count": 14, "max_ms": 87.152, "p95_ms": 87.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.847, "count": 14, "max_ms": 95.512, "p95_ms": 95.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.651, "count": 14, "max_ms": 121.226, "p95_ms": 121.226}, "read_interval": {"avg_ms": 2324.01, "count": 14, "max_ms": 15998.551, "p95_ms": 15998.551}, "triton_wait": {"avg_ms": 86.469, "count": 14, "max_ms": 108.407, "p95_ms": 108.407}, "visualize": {"avg_ms": 3.729, "count": 14, "max_ms": 4.685, "p95_ms": 4.685}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.458, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2822}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6116.844, "limit_mb": 8796093022207.996}, "detail": {"pid": 36514, "restart_count": 94, "start_count": 95}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:54:38+09:00", "last_error_summary": "[h264 @ 0x5644c8db99c0] error while decoding MB 41 36, bytestream -5 [in#0/rtsp @ 0x5644c8740e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5644c8740e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-06T23:54:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:54:41+09:00", "pid": 36514, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.215, "name": "ffmpeg", "pid": 36514, "rss_mb": 58.215, "state": "S (sleeping)", "threads": 1}, "restart_count": 94, "restart_count_per_hour": 12, "rtsp_timeout_count": 148, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.507, "sample_count": 2096, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 338655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:54:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.958, "last_frame_pushed_at": "2026-07-06T23:54:38+09:00", "last_frame_read_age_sec": 5.063, "last_frame_read_at": "2026-07-06T23:54:38+09:00", "mjpeg": {"age_sec": 4.941, "exists": true, "mtime": "2026-07-06T23:54:38.949692+09:00", "mtime_epoch": 1783349678.949692, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 110233}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:54:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1574.164, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.625, "pushed": 0.625, "read": 0.625}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.419, "count": 14, "max_ms": 5.961, "p95_ms": 5.961}, "gstreamer_push": {"avg_ms": 1.274, "count": 14, "max_ms": 1.398, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 68.974, "count": 14, "max_ms": 85.822, "p95_ms": 85.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.427, "count": 14, "max_ms": 108.394, "p95_ms": 108.394}, "yolo11_640_fire_11class": {"avg_ms": 77.06, "count": 14, "max_ms": 87.152, "p95_ms": 87.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.847, "count": 14, "max_ms": 95.512, "p95_ms": 95.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.651, "count": 14, "max_ms": 121.226, "p95_ms": 121.226}, "read_interval": {"avg_ms": 2324.01, "count": 14, "max_ms": 15998.551, "p95_ms": 15998.551}, "triton_wait": {"avg_ms": 86.469, "count": 14, "max_ms": 108.407, "p95_ms": 108.407}, "visualize": {"avg_ms": 3.729, "count": 14, "max_ms": 4.685, "p95_ms": 4.685}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.403, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6105.762, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x563525f8e200] missing picture in access unit with size 18 [h264 @ 0x563526a079c0] no frame! [in#0/rtsp @ 0x563525f89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563525f89e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563525f8e840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:57:14+09:00", "last_error_summary": "[NULL @ 0x563525f8e200] missing picture in access unit with size 18 [h264 @ 0x563526a079c0] no frame! [in#0/rtsp @ 0x563525f89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563525f89e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563525f8e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T23:54:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.243, "last_started_at": "2026-07-06T23:54:41+09:00", "pid": 36514, "process": {"exists": false, "pid": 36514}, "restart_count": 94, "restart_count_per_hour": 10, "rtsp_timeout_count": 150, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.857, "sample_count": 2090, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 339730, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:57:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-06T23:57:14+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-06T23:57:14+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-06T23:57:14.073372+09:00", "mtime_epoch": 1783349834.0733716, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 108976}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:57:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1574.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 21, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.701, "pushed": 0.701, "read": 0.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.217, "count": 21, "max_ms": 4.948, "p95_ms": 4.456}, "gstreamer_push": {"avg_ms": 1.33, "count": 21, "max_ms": 1.91, "p95_ms": 1.392}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.119, "count": 21, "max_ms": 86.159, "p95_ms": 79.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.742, "count": 21, "max_ms": 92.79, "p95_ms": 88.448}, "yolo11_640_fire_11class": {"avg_ms": 77.449, "count": 21, "max_ms": 87.569, "p95_ms": 87.021}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.982, "count": 21, "max_ms": 101.312, "p95_ms": 98.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.926, "count": 21, "max_ms": 114.252, "p95_ms": 111.204}, "read_interval": {"avg_ms": 1429.034, "count": 21, "max_ms": 15848.356, "p95_ms": 12037.299}, "triton_wait": {"avg_ms": 87.025, "count": 21, "max_ms": 101.324, "p95_ms": 98.403}, "visualize": {"avg_ms": 3.708, "count": 21, "max_ms": 4.538, "p95_ms": 4.146}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6105.375, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-06T23:57:14+09:00", "last_error_summary": "[NULL @ 0x563525f8e200] missing picture in access unit with size 18 [h264 @ 0x563526a079c0] no frame! [in#0/rtsp @ 0x563525f89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563525f89e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563525f8e840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:57:14+09:00", "last_error_summary": "[NULL @ 0x563525f8e200] missing picture in access unit with size 18 [h264 @ 0x563526a079c0] no frame! [in#0/rtsp @ 0x563525f89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563525f89e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563525f8e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T23:57:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.285, "last_started_at": "2026-07-06T23:54:41+09:00", "pid": 36514, "process": {"exists": false, "pid": 36514}, "restart_count": 94, "restart_count_per_hour": 10, "rtsp_timeout_count": 150, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.857, "sample_count": 2090, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 339731, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:57:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-06T23:57:14+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-06T23:57:14+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-06T23:57:14.073372+09:00", "mtime_epoch": 1783349834.0733716, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 108976}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:57:14+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1574.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 20, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.702, "pushed": 0.702, "read": 0.669}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.209, "count": 21, "max_ms": 4.948, "p95_ms": 4.456}, "gstreamer_push": {"avg_ms": 1.331, "count": 21, "max_ms": 1.91, "p95_ms": 1.392}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.586, "count": 21, "max_ms": 86.159, "p95_ms": 79.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.933, "count": 21, "max_ms": 92.79, "p95_ms": 88.448}, "yolo11_640_fire_11class": {"avg_ms": 77.557, "count": 21, "max_ms": 87.569, "p95_ms": 87.021}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 86.007, "count": 21, "max_ms": 101.312, "p95_ms": 98.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.937, "count": 21, "max_ms": 114.252, "p95_ms": 111.204}, "read_interval": {"avg_ms": 1494.946, "count": 20, "max_ms": 15848.356, "p95_ms": 12037.299}, "triton_wait": {"avg_ms": 87.049, "count": 21, "max_ms": 101.324, "p95_ms": 98.403}, "visualize": {"avg_ms": 3.704, "count": 21, "max_ms": 4.538, "p95_ms": 4.146}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6105.359, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:57:14+09:00", "last_error_summary": "[NULL @ 0x563525f8e200] missing picture in access unit with size 18 [h264 @ 0x563526a079c0] no frame! [in#0/rtsp @ 0x563525f89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563525f89e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563525f8e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T23:57:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.292, "last_started_at": "2026-07-06T23:54:41+09:00", "pid": 36514, "process": {"exists": false, "pid": 36514}, "restart_count": 94, "restart_count_per_hour": 10, "rtsp_timeout_count": 150, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.857, "sample_count": 2090, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 339731, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:57:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-06T23:57:14+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-06T23:57:14+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-06T23:57:14.073372+09:00", "mtime_epoch": 1783349834.0733716, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 108976}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:57:14+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1574.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 20, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.702, "pushed": 0.702, "read": 0.669}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.209, "count": 21, "max_ms": 4.948, "p95_ms": 4.456}, "gstreamer_push": {"avg_ms": 1.331, "count": 21, "max_ms": 1.91, "p95_ms": 1.392}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 71.586, "count": 21, "max_ms": 86.159, "p95_ms": 79.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.933, "count": 21, "max_ms": 92.79, "p95_ms": 88.448}, "yolo11_640_fire_11class": {"avg_ms": 77.557, "count": 21, "max_ms": 87.569, "p95_ms": 87.021}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 86.007, "count": 21, "max_ms": 101.312, "p95_ms": 98.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.937, "count": 21, "max_ms": 114.252, "p95_ms": 111.204}, "read_interval": {"avg_ms": 1494.946, "count": 20, "max_ms": 15848.356, "p95_ms": 12037.299}, "triton_wait": {"avg_ms": 87.049, "count": 21, "max_ms": 101.324, "p95_ms": 98.403}, "visualize": {"avg_ms": 3.704, "count": 21, "max_ms": 4.538, "p95_ms": 4.146}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260706_124203_add025d3.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 2831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6122.176, "limit_mb": 8796093022207.996}, "detail": {"pid": 36666, "restart_count": 95, "start_count": 96}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-06T23:57:14+09:00", "last_error_summary": "[NULL @ 0x563525f8e200] missing picture in access unit with size 18 [h264 @ 0x563526a079c0] no frame! [in#0/rtsp @ 0x563525f89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563525f89e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563525f8e840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-06T23:57:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-06T23:57:16+09:00", "pid": 36666, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.996, "name": "ffmpeg", "pid": 36666, "rss_mb": 57.996, "state": "S (sleeping)", "threads": 1}, "restart_count": 95, "restart_count_per_hour": 11, "rtsp_timeout_count": 150, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 96}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.873, "sample_count": 2050, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 339731, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-06T23:57:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "add025d3", "last_frame_pushed_age_sec": 4.465, "last_frame_pushed_at": "2026-07-06T23:57:14+09:00", "last_frame_read_age_sec": 4.566, "last_frame_read_at": "2026-07-06T23:57:14+09:00", "mjpeg": {"age_sec": 4.449, "exists": true, "mtime": "2026-07-06T23:57:14.185377+09:00", "mtime_epoch": 1783349834.1853766, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 110339}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-06T23:57:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1626.184, "name": "celery", "pid": 267, "rss_mb": 1574.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.601, "pushed": 0.601, "read": 0.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.178, "count": 13, "max_ms": 4.455, "p95_ms": 4.455}, "gstreamer_push": {"avg_ms": 1.311, "count": 13, "max_ms": 1.392, "p95_ms": 1.392}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 70.731, "count": 13, "max_ms": 79.764, "p95_ms": 79.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.041, "count": 13, "max_ms": 92.79, "p95_ms": 92.79}, "yolo11_640_fire_11class": {"avg_ms": 77.429, "count": 13, "max_ms": 87.569, "p95_ms": 87.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 85.442, "count": 13, "max_ms": 101.312, "p95_ms": 101.312}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.72, "count": 13, "max_ms": 114.252, "p95_ms": 114.252}, "read_interval": {"avg_ms": 2238.469, "count": 13, "max_ms": 15848.356, "p95_ms": 15848.356}, "triton_wait": {"avg_ms": 86.748, "count": 13, "max_ms": 101.324, "p95_ms": 101.324}, "visualize": {"avg_ms": 3.751, "count": 13, "max_ms": 4.538, "p95_ms": 4.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.629, "window_s": 30.0}, "started_at": "2026-07-06T12:42:03+09:00", "state": "running", "task_id": "3fd35ee0-3eec-45d2-8995-adef798a2f40", "trigger": "start"}